From 7a27aa5d7bcc7a17354a96a1e121d566d177f322 Mon Sep 17 00:00:00 2001 From: Tarrence van As Date: Mon, 6 Nov 2023 09:01:27 -0500 Subject: [PATCH 01/16] Refactor torii grpc --- crates/dojo-types/src/schema.rs | 6 ++ crates/torii/client/src/client/mod.rs | 20 +++--- .../torii/client/src/client/subscription.rs | 29 ++++---- crates/torii/core/src/error.rs | 5 +- crates/torii/grpc/src/client.rs | 3 +- crates/torii/grpc/src/server/mod.rs | 14 +--- crates/torii/grpc/src/server/query.rs | 64 ++++++++++++++++++ crates/torii/grpc/src/server/subscription.rs | 42 ++++++++++-- .../grpc/src/{conversion.rs => types.rs} | 66 ++++++++++++++++++- 9 files changed, 199 insertions(+), 50 deletions(-) create mode 100644 crates/torii/grpc/src/server/query.rs rename crates/torii/grpc/src/{conversion.rs => types.rs} (81%) diff --git a/crates/dojo-types/src/schema.rs b/crates/dojo-types/src/schema.rs index 46cce85435..fbcde7d28f 100644 --- a/crates/dojo-types/src/schema.rs +++ b/crates/dojo-types/src/schema.rs @@ -20,6 +20,12 @@ impl Member { } } +#[derive(Debug, thiserror::Error)] +pub enum QueryError { + #[error("unsupported query")] + UnsupportedQuery, +} + #[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] pub struct EntityQuery { pub model: String, diff --git a/crates/torii/client/src/client/mod.rs b/crates/torii/client/src/client/mod.rs index 6c58f2c559..e4a342dcd8 100644 --- a/crates/torii/client/src/client/mod.rs +++ b/crates/torii/client/src/client/mod.rs @@ -7,7 +7,7 @@ use std::collections::HashSet; use std::sync::Arc; use dojo_types::packing::unpack; -use dojo_types::schema::{Clause, EntityQuery, Ty}; +use dojo_types::schema::Ty; use dojo_types::WorldMetadata; use dojo_world::contracts::WorldContractReader; use parking_lot::{RwLock, RwLockReadGuard}; @@ -16,7 +16,7 @@ use starknet::providers::jsonrpc::HttpTransport; use starknet::providers::JsonRpcClient; use starknet_crypto::FieldElement; use tokio::sync::RwLock as AsyncRwLock; -use torii_grpc::client::EntityUpdateStreaming; +use torii_grpc::client::{Clause, EntityUpdateStreaming, Query}; use self::error::{Error, ParseError}; use self::storage::ModelStorage; @@ -46,7 +46,7 @@ impl Client { torii_url: String, rpc_url: String, world: FieldElement, - queries: Option>, + queries: Option>, ) -> Result { let mut grpc_client = torii_grpc::client::WorldClient::new(torii_url, world).await?; @@ -66,7 +66,7 @@ impl Client { // TODO: change this to querying the gRPC url instead let subbed_entities = subbed_entities.entities.read().clone(); - for EntityQuery { model, clause } in subbed_entities { + for Query { model, clause } in subbed_entities { let model_reader = world_reader.model(&model).await?; let keys = if let Clause::Keys(clause) = clause { clause.keys @@ -98,7 +98,7 @@ impl Client { self.metadata.read() } - pub fn subscribed_entities(&self) -> RwLockReadGuard<'_, HashSet> { + pub fn subscribed_entities(&self) -> RwLockReadGuard<'_, HashSet> { self.subscribed_entities.entities.read() } @@ -109,7 +109,7 @@ impl Client { /// /// If the requested entity is not among the synced entities, it will attempt to fetch it from /// the RPC. - pub async fn entity(&self, entity: &EntityQuery) -> Result, Error> { + pub async fn entity(&self, entity: &Query) -> Result, Error> { let Some(mut schema) = self.metadata.read().model(&entity.model).map(|m| m.schema.clone()) else { return Ok(None); @@ -169,7 +169,7 @@ impl Client { /// Adds entities to the list of entities to be synced. /// /// NOTE: This will establish a new subscription stream with the server. - pub async fn add_entities_to_sync(&self, entities: Vec) -> Result<(), Error> { + pub async fn add_entities_to_sync(&self, entities: Vec) -> Result<(), Error> { for entity in &entities { let keys = if let Clause::Keys(clause) = entity.clone().clause { clause.keys @@ -196,7 +196,7 @@ impl Client { /// Removes entities from the list of entities to be synced. /// /// NOTE: This will establish a new subscription stream with the server. - pub async fn remove_entities_to_sync(&self, entities: Vec) -> Result<(), Error> { + pub async fn remove_entities_to_sync(&self, entities: Vec) -> Result<(), Error> { self.subscribed_entities.remove_entities(entities)?; let updated_entities = @@ -216,10 +216,10 @@ impl Client { async fn initiate_subscription( &self, - entities: Vec, + queries: Vec, ) -> Result { let mut grpc_client = self.inner.write().await; - let stream = grpc_client.subscribe_entities(entities).await?; + let stream = grpc_client.subscribe_entities(queries).await?; Ok(stream) } diff --git a/crates/torii/client/src/client/subscription.rs b/crates/torii/client/src/client/subscription.rs index 8bd82346f1..a6bd952317 100644 --- a/crates/torii/client/src/client/subscription.rs +++ b/crates/torii/client/src/client/subscription.rs @@ -4,7 +4,6 @@ use std::future::Future; use std::sync::Arc; use std::task::Poll; -use dojo_types::schema::{Clause, EntityQuery}; use dojo_types::WorldMetadata; use futures::channel::mpsc::{self, Receiver, Sender}; use futures_util::StreamExt; @@ -12,7 +11,7 @@ use parking_lot::{Mutex, RwLock}; use starknet::core::types::{StateDiff, StateUpdate}; use starknet::core::utils::cairo_short_string_to_felt; use starknet_crypto::FieldElement; -use torii_grpc::client::EntityUpdateStreaming; +use torii_grpc::client::{Clause, EntityUpdateStreaming, Query}; use super::error::{Error, ParseError}; use super::ModelStorage; @@ -24,13 +23,13 @@ pub enum SubscriptionEvent { pub struct SubscribedEntities { metadata: Arc>, - pub(super) entities: RwLock>, + pub(super) entities: RwLock>, /// All the relevant storage addresses derived from the subscribed entities pub(super) subscribed_storage_addresses: RwLock>, } impl SubscribedEntities { - pub(super) fn is_synced(&self, entity: &EntityQuery) -> bool { + pub(super) fn is_synced(&self, entity: &Query) -> bool { self.entities.read().contains(entity) } @@ -42,21 +41,21 @@ impl SubscribedEntities { } } - pub(super) fn add_entities(&self, entities: Vec) -> Result<(), Error> { + pub(super) fn add_entities(&self, entities: Vec) -> Result<(), Error> { for entity in entities { Self::add_entity(self, entity)?; } Ok(()) } - pub(super) fn remove_entities(&self, entities: Vec) -> Result<(), Error> { + pub(super) fn remove_entities(&self, entities: Vec) -> Result<(), Error> { for entity in entities { Self::remove_entity(self, entity)?; } Ok(()) } - pub(super) fn add_entity(&self, entity: EntityQuery) -> Result<(), Error> { + pub(super) fn add_entity(&self, entity: Query) -> Result<(), Error> { if !self.entities.write().insert(entity.clone()) { return Ok(()); } @@ -90,7 +89,7 @@ impl SubscribedEntities { Ok(()) } - pub(super) fn remove_entity(&self, entity: EntityQuery) -> Result<(), Error> { + pub(super) fn remove_entity(&self, entity: Query) -> Result<(), Error> { if !self.entities.write().remove(&entity) { return Ok(()); } @@ -206,7 +205,11 @@ impl SubscriptionService { let storage_entries = diff.storage_diffs.into_iter().find_map(|d| { let expected = self.world_metadata.read().world_address; let current = d.address; - if current == expected { Some(d.storage_entries) } else { None } + if current == expected { + Some(d.storage_entries) + } else { + None + } }); let Some(entries) = storage_entries else { @@ -256,11 +259,12 @@ mod tests { use std::collections::HashMap; use std::sync::Arc; - use dojo_types::schema::{KeysClause, Ty}; + use dojo_types::schema::Ty; use dojo_types::WorldMetadata; use parking_lot::RwLock; use starknet::core::utils::cairo_short_string_to_felt; use starknet::macros::felt; + use torii_grpc::client::{Clause, KeysClause, Query}; use crate::utils::compute_all_storage_addresses; @@ -295,10 +299,7 @@ mod tests { let metadata = self::create_dummy_metadata(); - let entity = dojo_types::schema::EntityQuery { - model: model_name, - clause: dojo_types::schema::Clause::Keys(KeysClause { keys }), - }; + let entity = Query { model: model_name, clause: Clause::Keys(KeysClause { keys }) }; let subscribed_entities = super::SubscribedEntities::new(Arc::new(RwLock::new(metadata))); subscribed_entities.add_entities(vec![entity.clone()]).expect("able to add entity"); diff --git a/crates/torii/core/src/error.rs b/crates/torii/core/src/error.rs index 8ccccdc601..737ec2d75a 100644 --- a/crates/torii/core/src/error.rs +++ b/crates/torii/core/src/error.rs @@ -1,3 +1,4 @@ +use dojo_types::schema::QueryError; use starknet::core::types::{FromByteSliceError, FromStrError}; use starknet::core::utils::CairoShortStringToFeltError; @@ -7,8 +8,8 @@ pub enum Error { Parse(#[from] ParseError), #[error(transparent)] Sql(#[from] sqlx::Error), - #[error("unsupported query clause")] - UnsupportedQuery, + #[error(transparent)] + QueryError(#[from] QueryError), } #[derive(Debug, thiserror::Error)] diff --git a/crates/torii/grpc/src/client.rs b/crates/torii/grpc/src/client.rs index a143a6788c..b829df721d 100644 --- a/crates/torii/grpc/src/client.rs +++ b/crates/torii/grpc/src/client.rs @@ -1,5 +1,4 @@ //! Client implementation for the gRPC service. - use futures_util::stream::MapOk; use futures_util::{Stream, StreamExt, TryStreamExt}; use protos::world::{world_client, SubscribeEntitiesRequest}; @@ -67,7 +66,7 @@ impl WorldClient { /// Subscribe to the state diff for a set of entities of a World. pub async fn subscribe_entities( &mut self, - queries: Vec, + queries: Vec, ) -> Result { let stream = self .inner diff --git a/crates/torii/grpc/src/server/mod.rs b/crates/torii/grpc/src/server/mod.rs index a9ca671f3a..120cf71359 100644 --- a/crates/torii/grpc/src/server/mod.rs +++ b/crates/torii/grpc/src/server/mod.rs @@ -1,5 +1,6 @@ pub mod error; pub mod logger; +pub mod query; pub mod subscription; use std::pin::Pin; @@ -141,17 +142,6 @@ impl DojoWorld { { let mut subs = Vec::with_capacity(queries.len()); for query in queries { - let clause: KeysClause = query - .clause - .ok_or(Error::UnsupportedQuery) - .and_then(|clause| clause.clause_type.ok_or(Error::UnsupportedQuery)) - .and_then(|clause_type| match clause_type { - ClauseType::Keys(clause) => Ok(clause), - _ => Err(Error::UnsupportedQuery), - })? - .try_into() - .map_err(ParseError::FromByteSliceError)?; - let model = cairo_short_string_to_felt(&query.model) .map_err(ParseError::CairoShortStringToFelt)?; @@ -159,7 +149,7 @@ impl DojoWorld { self.model_metadata(&query.model).await?; subs.push(SubscribeRequest { - keys: clause.keys, + query, model: subscription::ModelMetadata { name: model, packed_size: packed_size as usize, diff --git a/crates/torii/grpc/src/server/query.rs b/crates/torii/grpc/src/server/query.rs new file mode 100644 index 0000000000..511cac2824 --- /dev/null +++ b/crates/torii/grpc/src/server/query.rs @@ -0,0 +1,64 @@ +use serde::{Deserialize, Serialize}; +use starknet_crypto::FieldElement; + +#[derive(Debug, thiserror::Error)] +pub enum QueryError { + #[error("unsupported query")] + UnsupportedQuery, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] +pub struct Query { + pub model: String, + pub clause: Clause, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] +pub enum Clause { + Keys(KeysClause), + Attribute(AttributeClause), + Composite(CompositeClause), +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] +pub struct KeysClause { + pub keys: Vec, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] +pub struct AttributeClause { + pub attribute: String, + pub operator: ComparisonOperator, + pub value: Value, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] +pub struct CompositeClause { + pub operator: LogicalOperator, + pub clauses: Vec, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] +pub enum LogicalOperator { + And, + Or, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] +pub enum ComparisonOperator { + Eq, + Neq, + Gt, + Gte, + Lt, + Lte, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] +pub enum Value { + String(String), + Int(i64), + UInt(u64), + Bool(bool), + Bytes(Vec), +} diff --git a/crates/torii/grpc/src/server/subscription.rs b/crates/torii/grpc/src/server/subscription.rs index 5eb9d3176a..7a545377ff 100644 --- a/crates/torii/grpc/src/server/subscription.rs +++ b/crates/torii/grpc/src/server/subscription.rs @@ -3,6 +3,7 @@ use std::future::Future; use std::sync::Arc; use std::task::Poll; +use dojo_types::schema::EntityQuery; use futures_util::future::BoxFuture; use futures_util::FutureExt; use rand::Rng; @@ -27,7 +28,22 @@ pub struct ModelMetadata { pub struct SubscribeRequest { pub model: ModelMetadata, - pub keys: Vec, + pub query: EntityQuery, +} + +impl SubscribeRequest { + pub fn slots(&self) -> Result, QueryError> { + match self.query.clause { + Clause::Keys(KeysClause { keys }) => { + let base = poseidon_hash_many(&[ + short_string!("dojo_storage"), + req.model.name, + poseidon_hash_many(&keys), + ]); + } + _ => Err(QueryError::UnsupportedQuery), + } + } } pub struct Subscriber { @@ -45,23 +61,35 @@ pub struct SubscriberManager { impl SubscriberManager { pub(super) async fn add_subscriber( &self, - entities: Vec, + reqs: Vec, ) -> Receiver> { let id = rand::thread_rng().gen::(); let (sender, receiver) = channel(1); // convert the list of entites into a list storage addresses - let storage_addresses = entities + let storage_addresses = reqs .par_iter() - .map(|entity| { + .map(|req| { + let clause: KeysClause = req + .query + .clause + .ok_or(Error::UnsupportedQuery) + .and_then(|clause| clause.clause_type.ok_or(Error::UnsupportedQuery)) + .and_then(|clause_type| match clause_type { + ClauseType::Keys(clause) => Ok(clause), + _ => Err(Error::UnsupportedQuery), + })? + .try_into() + .map_err(ParseError::FromByteSliceError)?; + let base = poseidon_hash_many(&[ short_string!("dojo_storage"), - entity.model.name, - poseidon_hash_many(&entity.keys), + req.model.name, + poseidon_hash_many(&clause.keys), ]); - (0..entity.model.packed_size) + (0..req.model.packed_size) .into_par_iter() .map(|i| base + i.into()) .collect::>() diff --git a/crates/torii/grpc/src/conversion.rs b/crates/torii/grpc/src/types.rs similarity index 81% rename from crates/torii/grpc/src/conversion.rs rename to crates/torii/grpc/src/types.rs index 0ddbeadb78..0d7d9be85d 100644 --- a/crates/torii/grpc/src/conversion.rs +++ b/crates/torii/grpc/src/types.rs @@ -1,9 +1,7 @@ use std::collections::HashMap; use std::str::FromStr; -use dojo_types::schema::{ - AttributeClause, Clause, CompositeClause, EntityQuery, KeysClause, Ty, Value, -}; +use dojo_types::schema::Ty; use starknet::core::types::{ ContractStorageDiffItem, FromByteSliceError, FromStrError, StateDiff, StateUpdate, StorageEntry, }; @@ -11,6 +9,68 @@ use starknet_crypto::FieldElement; use crate::protos; +#[derive(Debug, thiserror::Error)] +pub enum QueryError { + #[error("unsupported query")] + UnsupportedQuery, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] +pub struct Query { + pub model: String, + pub clause: Clause, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] +pub enum Clause { + Keys(KeysClause), + Attribute(AttributeClause), + Composite(CompositeClause), +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] +pub struct KeysClause { + pub keys: Vec, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] +pub struct AttributeClause { + pub attribute: String, + pub operator: ComparisonOperator, + pub value: Value, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] +pub struct CompositeClause { + pub operator: LogicalOperator, + pub clauses: Vec, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] +pub enum LogicalOperator { + And, + Or, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] +pub enum ComparisonOperator { + Eq, + Neq, + Gt, + Gte, + Lt, + Lte, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] +pub enum Value { + String(String), + Int(i64), + UInt(u64), + Bool(bool), + Bytes(Vec), +} + impl TryFrom for dojo_types::schema::ModelMetadata { type Error = FromStrError; fn try_from(value: protos::types::ModelMetadata) -> Result { From a687c917275c0f43bdf67fcb2d69e0e7e3f15030 Mon Sep 17 00:00:00 2001 From: broody Date: Fri, 10 Nov 2023 12:55:54 -0800 Subject: [PATCH 02/16] error freeeee! --- crates/dojo-types/src/schema.rs | 62 ----------------- crates/torii/client/src/client/mod.rs | 3 +- crates/torii/client/src/client/storage.rs | 15 +++-- .../torii/client/src/client/subscription.rs | 5 +- crates/torii/core/src/error.rs | 7 +- .../graphql/src/tests/types-test/Scarb.lock | 4 +- crates/torii/grpc/src/client.rs | 1 + crates/torii/grpc/src/lib.rs | 2 +- crates/torii/grpc/src/server/mod.rs | 6 +- crates/torii/grpc/src/server/query.rs | 6 -- crates/torii/grpc/src/server/subscription.rs | 65 ++++++++++-------- crates/torii/grpc/src/types.rs | 67 +------------------ examples/spawn-and-move/Scarb.lock | 6 +- 13 files changed, 66 insertions(+), 183 deletions(-) diff --git a/crates/dojo-types/src/schema.rs b/crates/dojo-types/src/schema.rs index b17cf2ec7b..ec86efd39c 100644 --- a/crates/dojo-types/src/schema.rs +++ b/crates/dojo-types/src/schema.rs @@ -20,68 +20,6 @@ impl Member { } } -#[derive(Debug, thiserror::Error)] -pub enum QueryError { - #[error("unsupported query")] - UnsupportedQuery, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] -pub struct EntityQuery { - pub model: String, - pub clause: Clause, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] -pub enum Clause { - Keys(KeysClause), - Attribute(AttributeClause), - Composite(CompositeClause), -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] -pub struct KeysClause { - pub keys: Vec, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] -pub struct AttributeClause { - pub attribute: String, - pub operator: ComparisonOperator, - pub value: Value, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] -pub struct CompositeClause { - pub operator: LogicalOperator, - pub clauses: Vec, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] -pub enum LogicalOperator { - And, - Or, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] -pub enum ComparisonOperator { - Eq, - Neq, - Gt, - Gte, - Lt, - Lte, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] -pub enum Value { - String(String), - Int(i64), - UInt(u64), - Bool(bool), - Bytes(Vec), -} - #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ModelMetadata { pub schema: Ty, diff --git a/crates/torii/client/src/client/mod.rs b/crates/torii/client/src/client/mod.rs index e4a342dcd8..1455425a72 100644 --- a/crates/torii/client/src/client/mod.rs +++ b/crates/torii/client/src/client/mod.rs @@ -16,7 +16,8 @@ use starknet::providers::jsonrpc::HttpTransport; use starknet::providers::JsonRpcClient; use starknet_crypto::FieldElement; use tokio::sync::RwLock as AsyncRwLock; -use torii_grpc::client::{Clause, EntityUpdateStreaming, Query}; +use torii_grpc::client::EntityUpdateStreaming; +use torii_grpc::server::query::{Query, Clause}; use self::error::{Error, ParseError}; use self::storage::ModelStorage; diff --git a/crates/torii/client/src/client/storage.rs b/crates/torii/client/src/client/storage.rs index 2f486825b2..cf157a266d 100644 --- a/crates/torii/client/src/client/storage.rs +++ b/crates/torii/client/src/client/storage.rs @@ -168,11 +168,12 @@ mod tests { use std::collections::HashMap; use std::sync::Arc; - use dojo_types::schema::{KeysClause, Ty}; + use dojo_types::schema::Ty; use dojo_types::WorldMetadata; use parking_lot::RwLock; use starknet::core::utils::cairo_short_string_to_felt; use starknet::macros::felt; + use torii_grpc::server::query::{KeysClause, Clause, Query}; use crate::client::error::Error; use crate::utils::compute_all_storage_addresses; @@ -202,9 +203,9 @@ mod tests { fn err_if_set_values_too_many() { let storage = create_dummy_storage(); let keys = vec![felt!("0x12345")]; - let entity = dojo_types::schema::EntityQuery { + let entity = Query { model: "Position".into(), - clause: dojo_types::schema::Clause::Keys(KeysClause { keys: keys.clone() }), + clause: Clause::Keys(KeysClause { keys: keys.clone() }), }; let values = vec![felt!("1"), felt!("2"), felt!("3"), felt!("4"), felt!("5")]; @@ -222,9 +223,9 @@ mod tests { fn err_if_set_values_too_few() { let storage = create_dummy_storage(); let keys = vec![felt!("0x12345")]; - let entity = dojo_types::schema::EntityQuery { + let entity = Query { model: "Position".into(), - clause: dojo_types::schema::Clause::Keys(KeysClause { keys: keys.clone() }), + clause: Clause::Keys(KeysClause { keys: keys.clone() }), }; let values = vec![felt!("1"), felt!("2")]; @@ -242,9 +243,9 @@ mod tests { fn set_and_get_entity_value() { let storage = create_dummy_storage(); let keys = vec![felt!("0x12345")]; - let entity = dojo_types::schema::EntityQuery { + let entity = Query { model: "Position".into(), - clause: dojo_types::schema::Clause::Keys(KeysClause { keys: keys.clone() }), + clause: Clause::Keys(KeysClause { keys: keys.clone() }), }; assert!(storage.storage.read().is_empty(), "storage must be empty initially"); diff --git a/crates/torii/client/src/client/subscription.rs b/crates/torii/client/src/client/subscription.rs index a6bd952317..474e6cade4 100644 --- a/crates/torii/client/src/client/subscription.rs +++ b/crates/torii/client/src/client/subscription.rs @@ -11,7 +11,8 @@ use parking_lot::{Mutex, RwLock}; use starknet::core::types::{StateDiff, StateUpdate}; use starknet::core::utils::cairo_short_string_to_felt; use starknet_crypto::FieldElement; -use torii_grpc::client::{Clause, EntityUpdateStreaming, Query}; +use torii_grpc::client::EntityUpdateStreaming; +use torii_grpc::server::query::{Query, Clause}; use super::error::{Error, ParseError}; use super::ModelStorage; @@ -264,7 +265,7 @@ mod tests { use parking_lot::RwLock; use starknet::core::utils::cairo_short_string_to_felt; use starknet::macros::felt; - use torii_grpc::client::{Clause, KeysClause, Query}; + use torii_grpc::server::query::{Clause, KeysClause, Query}; use crate::utils::compute_all_storage_addresses; diff --git a/crates/torii/core/src/error.rs b/crates/torii/core/src/error.rs index 737ec2d75a..cdcf6b9b95 100644 --- a/crates/torii/core/src/error.rs +++ b/crates/torii/core/src/error.rs @@ -1,4 +1,3 @@ -use dojo_types::schema::QueryError; use starknet::core::types::{FromByteSliceError, FromStrError}; use starknet::core::utils::CairoShortStringToFeltError; @@ -21,3 +20,9 @@ pub enum ParseError { #[error(transparent)] FromByteSliceError(#[from] FromByteSliceError), } + +#[derive(Debug, thiserror::Error)] +pub enum QueryError { + #[error("unsupported query")] + UnsupportedQuery, +} diff --git a/crates/torii/graphql/src/tests/types-test/Scarb.lock b/crates/torii/graphql/src/tests/types-test/Scarb.lock index 2ca7018569..6ad1bf2c03 100644 --- a/crates/torii/graphql/src/tests/types-test/Scarb.lock +++ b/crates/torii/graphql/src/tests/types-test/Scarb.lock @@ -3,14 +3,14 @@ version = 1 [[package]] name = "dojo" -version = "0.3.6" +version = "0.3.10" dependencies = [ "dojo_plugin", ] [[package]] name = "dojo_plugin" -version = "0.3.6" +version = "0.3.10" [[package]] name = "types_test" diff --git a/crates/torii/grpc/src/client.rs b/crates/torii/grpc/src/client.rs index b375ebe2c7..53d55f93ca 100644 --- a/crates/torii/grpc/src/client.rs +++ b/crates/torii/grpc/src/client.rs @@ -7,6 +7,7 @@ use starknet_crypto::FieldElement; use crate::proto::world::{MetadataRequest, SubscribeEntitiesResponse}; use crate::proto::{self}; +use crate::server::query::Query; #[derive(Debug, thiserror::Error)] pub enum Error { diff --git a/crates/torii/grpc/src/lib.rs b/crates/torii/grpc/src/lib.rs index 4b19ce10cb..1fb5da7eb7 100644 --- a/crates/torii/grpc/src/lib.rs +++ b/crates/torii/grpc/src/lib.rs @@ -3,7 +3,7 @@ extern crate wasm_prost as prost; #[cfg(target_arch = "wasm32")] extern crate wasm_tonic as tonic; -pub mod conversion; +pub mod types; #[cfg(feature = "client")] pub mod client; diff --git a/crates/torii/grpc/src/server/mod.rs b/crates/torii/grpc/src/server/mod.rs index b37e745feb..6d426fbbce 100644 --- a/crates/torii/grpc/src/server/mod.rs +++ b/crates/torii/grpc/src/server/mod.rs @@ -8,7 +8,6 @@ use std::net::SocketAddr; use std::pin::Pin; use std::sync::Arc; -use dojo_types::schema::KeysClause; use futures::Stream; use proto::world::{ MetadataRequest, MetadataResponse, SubscribeEntitiesRequest, SubscribeEntitiesResponse, @@ -27,7 +26,6 @@ use torii_core::error::{Error, ParseError}; use torii_core::model::{parse_sql_model_members, SqlModelMember}; use self::subscription::SubscribeRequest; -use crate::proto::types::clause::ClauseType; use crate::proto::world::world_server::WorldServer; use crate::proto::{self}; @@ -162,9 +160,7 @@ impl DojoWorld { }); } - let res = self.subscriber_manager.add_subscriber(subs).await; - - Ok(res) + self.subscriber_manager.add_subscriber(subs).await } } diff --git a/crates/torii/grpc/src/server/query.rs b/crates/torii/grpc/src/server/query.rs index 511cac2824..bdfe304901 100644 --- a/crates/torii/grpc/src/server/query.rs +++ b/crates/torii/grpc/src/server/query.rs @@ -1,12 +1,6 @@ use serde::{Deserialize, Serialize}; use starknet_crypto::FieldElement; -#[derive(Debug, thiserror::Error)] -pub enum QueryError { - #[error("unsupported query")] - UnsupportedQuery, -} - #[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] pub struct Query { pub model: String, diff --git a/crates/torii/grpc/src/server/subscription.rs b/crates/torii/grpc/src/server/subscription.rs index 18b0c2e093..3aa264c1d5 100644 --- a/crates/torii/grpc/src/server/subscription.rs +++ b/crates/torii/grpc/src/server/subscription.rs @@ -3,11 +3,10 @@ use std::future::Future; use std::sync::Arc; use std::task::Poll; -use dojo_types::schema::EntityQuery; use futures_util::future::BoxFuture; use futures_util::FutureExt; use rand::Rng; -use rayon::prelude::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator}; +use rayon::prelude::{IntoParallelIterator, ParallelIterator}; use starknet::core::types::{ BlockId, ContractStorageDiffItem, MaybePendingStateUpdate, StateUpdate, StorageEntry, }; @@ -16,10 +15,13 @@ use starknet::providers::Provider; use starknet_crypto::{poseidon_hash_many, FieldElement}; use tokio::sync::mpsc::{channel, Receiver, Sender}; use tokio::sync::RwLock; +use torii_core::error::{Error, ParseError, QueryError}; use tracing::{debug, error, trace}; -use super::error::SubscriptionError as Error; +use super::error::SubscriptionError; use crate::proto; +use crate::proto::types::EntityQuery; +use super::query::KeysClause; pub struct ModelMetadata { pub name: FieldElement, @@ -32,18 +34,18 @@ pub struct SubscribeRequest { } impl SubscribeRequest { - pub fn slots(&self) -> Result, QueryError> { - match self.query.clause { - Clause::Keys(KeysClause { keys }) => { - let base = poseidon_hash_many(&[ - short_string!("dojo_storage"), - req.model.name, - poseidon_hash_many(&keys), - ]); - } - _ => Err(QueryError::UnsupportedQuery), - } - } + // pub fn slots(&self) -> Result, QueryError> { + // match self.query.clause { + // Clause::Keys(KeysClause { keys }) => { + // let base = poseidon_hash_many(&[ + // short_string!("dojo_storage"), + // req.model.name, + // poseidon_hash_many(&keys), + // ]); + // } + // _ => Err(QueryError::UnsupportedQuery), + // } + // } } pub struct Subscriber { @@ -62,44 +64,49 @@ impl SubscriberManager { pub(super) async fn add_subscriber( &self, reqs: Vec, - ) -> Receiver> { + ) -> Result>, Error> { let id = rand::thread_rng().gen::(); let (sender, receiver) = channel(1); // convert the list of entites into a list storage addresses let storage_addresses = reqs - .par_iter() + .into_iter() .map(|req| { let clause: KeysClause = req .query .clause - .ok_or(Error::UnsupportedQuery) - .and_then(|clause| clause.clause_type.ok_or(Error::UnsupportedQuery)) + .ok_or(QueryError::UnsupportedQuery) + .and_then(|clause| clause.clause_type.ok_or(QueryError::UnsupportedQuery)) .and_then(|clause_type| match clause_type { - ClauseType::Keys(clause) => Ok(clause), - _ => Err(Error::UnsupportedQuery), - })? + proto::types::clause::ClauseType::Keys(clause) => Ok(clause), + _ => Err(QueryError::UnsupportedQuery) + }) + .map_err(Error::QueryError)? .try_into() .map_err(ParseError::FromByteSliceError)?; let base = poseidon_hash_many(&[ short_string!("dojo_storage"), - req.model.name, + req.model.name, poseidon_hash_many(&clause.keys), ]); - (0..req.model.packed_size) + let res = (0..req.model.packed_size) .into_par_iter() .map(|i| base + i.into()) - .collect::>() + .collect::>(); + + Ok(res) }) + .collect::, Error>>()? + .into_iter() .flatten() .collect::>(); self.subscribers.write().await.insert(id, Subscriber { storage_addresses, sender }); - receiver + Ok(receiver) } pub(super) async fn remove_subscriber(&self, id: usize) { @@ -107,8 +114,8 @@ impl SubscriberManager { } } -type PublishStateUpdateResult = Result<(), Error>; -type RequestStateUpdateResult = Result; +type PublishStateUpdateResult = Result<(), SubscriptionError>; +type RequestStateUpdateResult = Result; #[must_use = "Service does nothing unless polled"] pub struct Service { @@ -144,7 +151,7 @@ where async fn fetch_state_update(provider: P, block_num: u64) -> (P, u64, RequestStateUpdateResult) { let res = - provider.get_state_update(BlockId::Number(block_num)).await.map_err(Error::Provider); + provider.get_state_update(BlockId::Number(block_num)).await.map_err(SubscriptionError::Provider); (provider, block_num, res) } diff --git a/crates/torii/grpc/src/types.rs b/crates/torii/grpc/src/types.rs index f63abc5b2e..09be665143 100644 --- a/crates/torii/grpc/src/types.rs +++ b/crates/torii/grpc/src/types.rs @@ -8,68 +8,7 @@ use starknet::core::types::{ use starknet_crypto::FieldElement; use crate::proto; - -#[derive(Debug, thiserror::Error)] -pub enum QueryError { - #[error("unsupported query")] - UnsupportedQuery, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] -pub struct Query { - pub model: String, - pub clause: Clause, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] -pub enum Clause { - Keys(KeysClause), - Attribute(AttributeClause), - Composite(CompositeClause), -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] -pub struct KeysClause { - pub keys: Vec, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] -pub struct AttributeClause { - pub attribute: String, - pub operator: ComparisonOperator, - pub value: Value, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] -pub struct CompositeClause { - pub operator: LogicalOperator, - pub clauses: Vec, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] -pub enum LogicalOperator { - And, - Or, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] -pub enum ComparisonOperator { - Eq, - Neq, - Gt, - Gte, - Lt, - Lte, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] -pub enum Value { - String(String), - Int(i64), - UInt(u64), - Bool(bool), - Bytes(Vec), -} +use crate::server::query::{Query, Clause, KeysClause, AttributeClause, CompositeClause, Value}; impl TryFrom for dojo_types::schema::ModelMetadata { type Error = FromStrError; @@ -106,8 +45,8 @@ impl TryFrom for dojo_types::WorldMetadata { } } -impl From for proto::types::EntityQuery { - fn from(value: EntityQuery) -> Self { +impl From for proto::types::EntityQuery { + fn from(value: Query) -> Self { Self { model: value.model, clause: Some(value.clause.into()) } } } diff --git a/examples/spawn-and-move/Scarb.lock b/examples/spawn-and-move/Scarb.lock index 9ffa906f1d..2370c6221a 100644 --- a/examples/spawn-and-move/Scarb.lock +++ b/examples/spawn-and-move/Scarb.lock @@ -3,18 +3,18 @@ version = 1 [[package]] name = "dojo" -version = "0.3.6" +version = "0.3.10" dependencies = [ "dojo_plugin", ] [[package]] name = "dojo_examples" -version = "0.3.6" +version = "0.3.10" dependencies = [ "dojo", ] [[package]] name = "dojo_plugin" -version = "0.3.6" +version = "0.3.10" From 7916515916cf68eb460b6143344be43380c5bcd4 Mon Sep 17 00:00:00 2001 From: Tarrence van As Date: Mon, 6 Nov 2023 09:01:27 -0500 Subject: [PATCH 03/16] Refactor torii grpc --- crates/dojo-types/src/schema.rs | 6 ++ crates/torii/client/src/client/mod.rs | 20 +++--- .../torii/client/src/client/subscription.rs | 29 +++++---- crates/torii/core/src/error.rs | 5 +- crates/torii/grpc/src/client.rs | 3 +- crates/torii/grpc/src/server/mod.rs | 14 +--- crates/torii/grpc/src/server/query.rs | 64 +++++++++++++++++++ crates/torii/grpc/src/server/subscription.rs | 41 ++++++++++-- .../grpc/src/{conversion.rs => types.rs} | 4 +- 9 files changed, 137 insertions(+), 49 deletions(-) create mode 100644 crates/torii/grpc/src/server/query.rs rename crates/torii/grpc/src/{conversion.rs => types.rs} (98%) diff --git a/crates/dojo-types/src/schema.rs b/crates/dojo-types/src/schema.rs index a6ff30ebc5..b17cf2ec7b 100644 --- a/crates/dojo-types/src/schema.rs +++ b/crates/dojo-types/src/schema.rs @@ -20,6 +20,12 @@ impl Member { } } +#[derive(Debug, thiserror::Error)] +pub enum QueryError { + #[error("unsupported query")] + UnsupportedQuery, +} + #[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] pub struct EntityQuery { pub model: String, diff --git a/crates/torii/client/src/client/mod.rs b/crates/torii/client/src/client/mod.rs index 6c58f2c559..e4a342dcd8 100644 --- a/crates/torii/client/src/client/mod.rs +++ b/crates/torii/client/src/client/mod.rs @@ -7,7 +7,7 @@ use std::collections::HashSet; use std::sync::Arc; use dojo_types::packing::unpack; -use dojo_types::schema::{Clause, EntityQuery, Ty}; +use dojo_types::schema::Ty; use dojo_types::WorldMetadata; use dojo_world::contracts::WorldContractReader; use parking_lot::{RwLock, RwLockReadGuard}; @@ -16,7 +16,7 @@ use starknet::providers::jsonrpc::HttpTransport; use starknet::providers::JsonRpcClient; use starknet_crypto::FieldElement; use tokio::sync::RwLock as AsyncRwLock; -use torii_grpc::client::EntityUpdateStreaming; +use torii_grpc::client::{Clause, EntityUpdateStreaming, Query}; use self::error::{Error, ParseError}; use self::storage::ModelStorage; @@ -46,7 +46,7 @@ impl Client { torii_url: String, rpc_url: String, world: FieldElement, - queries: Option>, + queries: Option>, ) -> Result { let mut grpc_client = torii_grpc::client::WorldClient::new(torii_url, world).await?; @@ -66,7 +66,7 @@ impl Client { // TODO: change this to querying the gRPC url instead let subbed_entities = subbed_entities.entities.read().clone(); - for EntityQuery { model, clause } in subbed_entities { + for Query { model, clause } in subbed_entities { let model_reader = world_reader.model(&model).await?; let keys = if let Clause::Keys(clause) = clause { clause.keys @@ -98,7 +98,7 @@ impl Client { self.metadata.read() } - pub fn subscribed_entities(&self) -> RwLockReadGuard<'_, HashSet> { + pub fn subscribed_entities(&self) -> RwLockReadGuard<'_, HashSet> { self.subscribed_entities.entities.read() } @@ -109,7 +109,7 @@ impl Client { /// /// If the requested entity is not among the synced entities, it will attempt to fetch it from /// the RPC. - pub async fn entity(&self, entity: &EntityQuery) -> Result, Error> { + pub async fn entity(&self, entity: &Query) -> Result, Error> { let Some(mut schema) = self.metadata.read().model(&entity.model).map(|m| m.schema.clone()) else { return Ok(None); @@ -169,7 +169,7 @@ impl Client { /// Adds entities to the list of entities to be synced. /// /// NOTE: This will establish a new subscription stream with the server. - pub async fn add_entities_to_sync(&self, entities: Vec) -> Result<(), Error> { + pub async fn add_entities_to_sync(&self, entities: Vec) -> Result<(), Error> { for entity in &entities { let keys = if let Clause::Keys(clause) = entity.clone().clause { clause.keys @@ -196,7 +196,7 @@ impl Client { /// Removes entities from the list of entities to be synced. /// /// NOTE: This will establish a new subscription stream with the server. - pub async fn remove_entities_to_sync(&self, entities: Vec) -> Result<(), Error> { + pub async fn remove_entities_to_sync(&self, entities: Vec) -> Result<(), Error> { self.subscribed_entities.remove_entities(entities)?; let updated_entities = @@ -216,10 +216,10 @@ impl Client { async fn initiate_subscription( &self, - entities: Vec, + queries: Vec, ) -> Result { let mut grpc_client = self.inner.write().await; - let stream = grpc_client.subscribe_entities(entities).await?; + let stream = grpc_client.subscribe_entities(queries).await?; Ok(stream) } diff --git a/crates/torii/client/src/client/subscription.rs b/crates/torii/client/src/client/subscription.rs index 8bd82346f1..a6bd952317 100644 --- a/crates/torii/client/src/client/subscription.rs +++ b/crates/torii/client/src/client/subscription.rs @@ -4,7 +4,6 @@ use std::future::Future; use std::sync::Arc; use std::task::Poll; -use dojo_types::schema::{Clause, EntityQuery}; use dojo_types::WorldMetadata; use futures::channel::mpsc::{self, Receiver, Sender}; use futures_util::StreamExt; @@ -12,7 +11,7 @@ use parking_lot::{Mutex, RwLock}; use starknet::core::types::{StateDiff, StateUpdate}; use starknet::core::utils::cairo_short_string_to_felt; use starknet_crypto::FieldElement; -use torii_grpc::client::EntityUpdateStreaming; +use torii_grpc::client::{Clause, EntityUpdateStreaming, Query}; use super::error::{Error, ParseError}; use super::ModelStorage; @@ -24,13 +23,13 @@ pub enum SubscriptionEvent { pub struct SubscribedEntities { metadata: Arc>, - pub(super) entities: RwLock>, + pub(super) entities: RwLock>, /// All the relevant storage addresses derived from the subscribed entities pub(super) subscribed_storage_addresses: RwLock>, } impl SubscribedEntities { - pub(super) fn is_synced(&self, entity: &EntityQuery) -> bool { + pub(super) fn is_synced(&self, entity: &Query) -> bool { self.entities.read().contains(entity) } @@ -42,21 +41,21 @@ impl SubscribedEntities { } } - pub(super) fn add_entities(&self, entities: Vec) -> Result<(), Error> { + pub(super) fn add_entities(&self, entities: Vec) -> Result<(), Error> { for entity in entities { Self::add_entity(self, entity)?; } Ok(()) } - pub(super) fn remove_entities(&self, entities: Vec) -> Result<(), Error> { + pub(super) fn remove_entities(&self, entities: Vec) -> Result<(), Error> { for entity in entities { Self::remove_entity(self, entity)?; } Ok(()) } - pub(super) fn add_entity(&self, entity: EntityQuery) -> Result<(), Error> { + pub(super) fn add_entity(&self, entity: Query) -> Result<(), Error> { if !self.entities.write().insert(entity.clone()) { return Ok(()); } @@ -90,7 +89,7 @@ impl SubscribedEntities { Ok(()) } - pub(super) fn remove_entity(&self, entity: EntityQuery) -> Result<(), Error> { + pub(super) fn remove_entity(&self, entity: Query) -> Result<(), Error> { if !self.entities.write().remove(&entity) { return Ok(()); } @@ -206,7 +205,11 @@ impl SubscriptionService { let storage_entries = diff.storage_diffs.into_iter().find_map(|d| { let expected = self.world_metadata.read().world_address; let current = d.address; - if current == expected { Some(d.storage_entries) } else { None } + if current == expected { + Some(d.storage_entries) + } else { + None + } }); let Some(entries) = storage_entries else { @@ -256,11 +259,12 @@ mod tests { use std::collections::HashMap; use std::sync::Arc; - use dojo_types::schema::{KeysClause, Ty}; + use dojo_types::schema::Ty; use dojo_types::WorldMetadata; use parking_lot::RwLock; use starknet::core::utils::cairo_short_string_to_felt; use starknet::macros::felt; + use torii_grpc::client::{Clause, KeysClause, Query}; use crate::utils::compute_all_storage_addresses; @@ -295,10 +299,7 @@ mod tests { let metadata = self::create_dummy_metadata(); - let entity = dojo_types::schema::EntityQuery { - model: model_name, - clause: dojo_types::schema::Clause::Keys(KeysClause { keys }), - }; + let entity = Query { model: model_name, clause: Clause::Keys(KeysClause { keys }) }; let subscribed_entities = super::SubscribedEntities::new(Arc::new(RwLock::new(metadata))); subscribed_entities.add_entities(vec![entity.clone()]).expect("able to add entity"); diff --git a/crates/torii/core/src/error.rs b/crates/torii/core/src/error.rs index 8ccccdc601..737ec2d75a 100644 --- a/crates/torii/core/src/error.rs +++ b/crates/torii/core/src/error.rs @@ -1,3 +1,4 @@ +use dojo_types::schema::QueryError; use starknet::core::types::{FromByteSliceError, FromStrError}; use starknet::core::utils::CairoShortStringToFeltError; @@ -7,8 +8,8 @@ pub enum Error { Parse(#[from] ParseError), #[error(transparent)] Sql(#[from] sqlx::Error), - #[error("unsupported query clause")] - UnsupportedQuery, + #[error(transparent)] + QueryError(#[from] QueryError), } #[derive(Debug, thiserror::Error)] diff --git a/crates/torii/grpc/src/client.rs b/crates/torii/grpc/src/client.rs index c97b250b16..b375ebe2c7 100644 --- a/crates/torii/grpc/src/client.rs +++ b/crates/torii/grpc/src/client.rs @@ -1,5 +1,4 @@ //! Client implementation for the gRPC service. - use futures_util::stream::MapOk; use futures_util::{Stream, StreamExt, TryStreamExt}; use proto::world::{world_client, SubscribeEntitiesRequest}; @@ -67,7 +66,7 @@ impl WorldClient { /// Subscribe to the state diff for a set of entities of a World. pub async fn subscribe_entities( &mut self, - queries: Vec, + queries: Vec, ) -> Result { let stream = self .inner diff --git a/crates/torii/grpc/src/server/mod.rs b/crates/torii/grpc/src/server/mod.rs index c4e2088e70..b37e745feb 100644 --- a/crates/torii/grpc/src/server/mod.rs +++ b/crates/torii/grpc/src/server/mod.rs @@ -1,5 +1,6 @@ pub mod error; pub mod logger; +pub mod query; pub mod subscription; use std::future::Future; @@ -146,17 +147,6 @@ impl DojoWorld { { let mut subs = Vec::with_capacity(queries.len()); for query in queries { - let clause: KeysClause = query - .clause - .ok_or(Error::UnsupportedQuery) - .and_then(|clause| clause.clause_type.ok_or(Error::UnsupportedQuery)) - .and_then(|clause_type| match clause_type { - ClauseType::Keys(clause) => Ok(clause), - _ => Err(Error::UnsupportedQuery), - })? - .try_into() - .map_err(ParseError::FromByteSliceError)?; - let model = cairo_short_string_to_felt(&query.model) .map_err(ParseError::CairoShortStringToFelt)?; @@ -164,7 +154,7 @@ impl DojoWorld { self.model_metadata(&query.model).await?; subs.push(SubscribeRequest { - keys: clause.keys, + query, model: subscription::ModelMetadata { name: model, packed_size: packed_size as usize, diff --git a/crates/torii/grpc/src/server/query.rs b/crates/torii/grpc/src/server/query.rs new file mode 100644 index 0000000000..511cac2824 --- /dev/null +++ b/crates/torii/grpc/src/server/query.rs @@ -0,0 +1,64 @@ +use serde::{Deserialize, Serialize}; +use starknet_crypto::FieldElement; + +#[derive(Debug, thiserror::Error)] +pub enum QueryError { + #[error("unsupported query")] + UnsupportedQuery, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] +pub struct Query { + pub model: String, + pub clause: Clause, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] +pub enum Clause { + Keys(KeysClause), + Attribute(AttributeClause), + Composite(CompositeClause), +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] +pub struct KeysClause { + pub keys: Vec, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] +pub struct AttributeClause { + pub attribute: String, + pub operator: ComparisonOperator, + pub value: Value, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] +pub struct CompositeClause { + pub operator: LogicalOperator, + pub clauses: Vec, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] +pub enum LogicalOperator { + And, + Or, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] +pub enum ComparisonOperator { + Eq, + Neq, + Gt, + Gte, + Lt, + Lte, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] +pub enum Value { + String(String), + Int(i64), + UInt(u64), + Bool(bool), + Bytes(Vec), +} diff --git a/crates/torii/grpc/src/server/subscription.rs b/crates/torii/grpc/src/server/subscription.rs index 4750d69cdd..f7cbc1e435 100644 --- a/crates/torii/grpc/src/server/subscription.rs +++ b/crates/torii/grpc/src/server/subscription.rs @@ -3,6 +3,7 @@ use std::future::Future; use std::sync::Arc; use std::task::Poll; +use dojo_types::schema::EntityQuery; use futures_util::future::BoxFuture; use futures_util::FutureExt; use rand::Rng; @@ -27,7 +28,22 @@ pub struct ModelMetadata { pub struct SubscribeRequest { pub model: ModelMetadata, - pub keys: Vec, + pub query: EntityQuery, +} + +impl SubscribeRequest { + pub fn slots(&self) -> Result, QueryError> { + match self.query.clause { + Clause::Keys(KeysClause { keys }) => { + let base = poseidon_hash_many(&[ + short_string!("dojo_storage"), + req.model.name, + poseidon_hash_many(&keys), + ]); + } + _ => Err(QueryError::UnsupportedQuery), + } + } } pub struct Subscriber { @@ -47,21 +63,34 @@ impl SubscriberManager { &self, entities: Vec, ) -> Receiver> { + let id = rand::thread_rng().gen::(); let (sender, receiver) = channel(1); // convert the list of entites into a list storage addresses - let storage_addresses = entities + let storage_addresses = reqs .par_iter() - .map(|entity| { + .map(|req| { + let clause: KeysClause = req + .query + .clause + .ok_or(Error::UnsupportedQuery) + .and_then(|clause| clause.clause_type.ok_or(Error::UnsupportedQuery)) + .and_then(|clause_type| match clause_type { + ClauseType::Keys(clause) => Ok(clause), + _ => Err(Error::UnsupportedQuery), + })? + .try_into() + .map_err(ParseError::FromByteSliceError)?; + let base = poseidon_hash_many(&[ short_string!("dojo_storage"), - entity.model.name, - poseidon_hash_many(&entity.keys), + req.model.name, + poseidon_hash_many(&clause.keys), ]); - (0..entity.model.packed_size) + (0..req.model.packed_size) .into_par_iter() .map(|i| base + i.into()) .collect::>() diff --git a/crates/torii/grpc/src/conversion.rs b/crates/torii/grpc/src/types.rs similarity index 98% rename from crates/torii/grpc/src/conversion.rs rename to crates/torii/grpc/src/types.rs index 987059cfb2..d5eb67964f 100644 --- a/crates/torii/grpc/src/conversion.rs +++ b/crates/torii/grpc/src/types.rs @@ -1,9 +1,7 @@ use std::collections::HashMap; use std::str::FromStr; -use dojo_types::schema::{ - AttributeClause, Clause, CompositeClause, EntityQuery, KeysClause, Ty, Value, -}; +use dojo_types::schema::Ty; use starknet::core::types::{ ContractStorageDiffItem, FromByteSliceError, FromStrError, StateDiff, StateUpdate, StorageEntry, }; From e5b32cc1e8704a7c1b3b835a4e153cc41486aa17 Mon Sep 17 00:00:00 2001 From: Tarrence van As Date: Mon, 6 Nov 2023 09:01:27 -0500 Subject: [PATCH 04/16] rebase changes --- .../torii/client/src/client/subscription.rs | 6 +- crates/torii/core/src/error.rs | 1 + crates/torii/grpc/src/server/mod.rs | 1 + crates/torii/grpc/src/server/query.rs | 64 +++++++++++++++++++ crates/torii/grpc/src/server/subscription.rs | 1 + 5 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 crates/torii/grpc/src/server/query.rs diff --git a/crates/torii/client/src/client/subscription.rs b/crates/torii/client/src/client/subscription.rs index 531ead079e..6c95a20f4f 100644 --- a/crates/torii/client/src/client/subscription.rs +++ b/crates/torii/client/src/client/subscription.rs @@ -206,7 +206,11 @@ impl SubscriptionService { let storage_entries = diff.storage_diffs.into_iter().find_map(|d| { let expected = self.world_metadata.read().world_address; let current = d.address; - if current == expected { Some(d.storage_entries) } else { None } + if current == expected { + Some(d.storage_entries) + } else { + None + } }); let Some(entries) = storage_entries else { diff --git a/crates/torii/core/src/error.rs b/crates/torii/core/src/error.rs index cdcf6b9b95..49da62fbcd 100644 --- a/crates/torii/core/src/error.rs +++ b/crates/torii/core/src/error.rs @@ -1,3 +1,4 @@ +use dojo_types::schema::QueryError; use starknet::core::types::{FromByteSliceError, FromStrError}; use starknet::core::utils::CairoShortStringToFeltError; diff --git a/crates/torii/grpc/src/server/mod.rs b/crates/torii/grpc/src/server/mod.rs index 7d226559a1..6d426fbbce 100644 --- a/crates/torii/grpc/src/server/mod.rs +++ b/crates/torii/grpc/src/server/mod.rs @@ -1,5 +1,6 @@ pub mod error; pub mod logger; +pub mod query; pub mod subscription; use std::future::Future; diff --git a/crates/torii/grpc/src/server/query.rs b/crates/torii/grpc/src/server/query.rs new file mode 100644 index 0000000000..511cac2824 --- /dev/null +++ b/crates/torii/grpc/src/server/query.rs @@ -0,0 +1,64 @@ +use serde::{Deserialize, Serialize}; +use starknet_crypto::FieldElement; + +#[derive(Debug, thiserror::Error)] +pub enum QueryError { + #[error("unsupported query")] + UnsupportedQuery, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] +pub struct Query { + pub model: String, + pub clause: Clause, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] +pub enum Clause { + Keys(KeysClause), + Attribute(AttributeClause), + Composite(CompositeClause), +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] +pub struct KeysClause { + pub keys: Vec, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] +pub struct AttributeClause { + pub attribute: String, + pub operator: ComparisonOperator, + pub value: Value, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] +pub struct CompositeClause { + pub operator: LogicalOperator, + pub clauses: Vec, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] +pub enum LogicalOperator { + And, + Or, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] +pub enum ComparisonOperator { + Eq, + Neq, + Gt, + Gte, + Lt, + Lte, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] +pub enum Value { + String(String), + Int(i64), + UInt(u64), + Bool(bool), + Bytes(Vec), +} diff --git a/crates/torii/grpc/src/server/subscription.rs b/crates/torii/grpc/src/server/subscription.rs index 983451fec4..fe15c7577a 100644 --- a/crates/torii/grpc/src/server/subscription.rs +++ b/crates/torii/grpc/src/server/subscription.rs @@ -3,6 +3,7 @@ use std::future::Future; use std::sync::Arc; use std::task::Poll; +use dojo_types::schema::EntityQuery; use futures_util::future::BoxFuture; use futures_util::FutureExt; use rand::Rng; From 36e5b18445e25504a928125abae70ff354708c06 Mon Sep 17 00:00:00 2001 From: broody Date: Fri, 10 Nov 2023 15:25:59 -0800 Subject: [PATCH 05/16] clean up --- crates/torii/client/src/client/mod.rs | 29 ++++----- crates/torii/client/src/client/storage.rs | 14 ++-- .../torii/client/src/client/subscription.rs | 29 ++++----- crates/torii/core/src/error.rs | 1 - crates/torii/grpc/src/server/mod.rs | 1 - crates/torii/grpc/src/server/query.rs | 64 ------------------- crates/torii/grpc/src/server/subscription.rs | 1 - 7 files changed, 33 insertions(+), 106 deletions(-) delete mode 100644 crates/torii/grpc/src/server/query.rs diff --git a/crates/torii/client/src/client/mod.rs b/crates/torii/client/src/client/mod.rs index 4508833abd..d14ade458f 100644 --- a/crates/torii/client/src/client/mod.rs +++ b/crates/torii/client/src/client/mod.rs @@ -110,26 +110,25 @@ impl Client { /// /// If the requested entity is not among the synced entities, it will attempt to fetch it from /// the RPC. - pub async fn entity(&self, entity: &Query) -> Result, Error> { - let Some(mut schema) = self.metadata.read().model(&entity.model).map(|m| m.schema.clone()) + pub async fn entity(&self, query: &Query) -> Result, Error> { + let Some(mut schema) = self.metadata.read().model(&query.model).map(|m| m.schema.clone()) else { return Ok(None); }; - let keys = if let Clause::Keys(clause) = entity.clone().clause { + let keys = if let Clause::Keys(clause) = query.clone().clause { clause.keys } else { return Err(Error::UnsupportedQuery); }; - if !self.subscribed_entities.is_synced(entity) { - let model = self.world_reader.model(&entity.model).await?; + if !self.subscribed_entities.is_synced(query) { + let model = self.world_reader.model(&query.model).await?; return Ok(Some(model.entity(&keys).await?)); } let Ok(Some(raw_values)) = self.storage.get_entity_storage( - cairo_short_string_to_felt(&entity.model) - .map_err(ParseError::CairoShortStringToFelt)?, + cairo_short_string_to_felt(&query.model).map_err(ParseError::CairoShortStringToFelt)?, &keys, ) else { return Ok(Some(schema)); @@ -138,7 +137,7 @@ impl Client { let layout = self .metadata .read() - .model(&entity.model) + .model(&query.model) .map(|m| m.layout.clone()) .expect("qed; layout should exist"); @@ -170,18 +169,18 @@ impl Client { /// Adds entities to the list of entities to be synced. /// /// NOTE: This will establish a new subscription stream with the server. - pub async fn add_entities_to_sync(&self, entities: Vec) -> Result<(), Error> { - for entity in &entities { - let keys = if let Clause::Keys(clause) = entity.clone().clause { + pub async fn add_entities_to_sync(&self, queries: Vec) -> Result<(), Error> { + for query in &queries { + let keys = if let Clause::Keys(clause) = query.clone().clause { clause.keys } else { return Err(Error::UnsupportedQuery); }; - self.initiate_entity(&entity.model, keys.clone()).await?; + self.initiate_entity(&query.model, keys.clone()).await?; } - self.subscribed_entities.add_entities(entities)?; + self.subscribed_entities.add_entities(queries)?; let updated_entities = self.subscribed_entities.entities.read().clone().into_iter().collect(); @@ -197,8 +196,8 @@ impl Client { /// Removes entities from the list of entities to be synced. /// /// NOTE: This will establish a new subscription stream with the server. - pub async fn remove_entities_to_sync(&self, entities: Vec) -> Result<(), Error> { - self.subscribed_entities.remove_entities(entities)?; + pub async fn remove_entities_to_sync(&self, queries: Vec) -> Result<(), Error> { + self.subscribed_entities.remove_entities(queries)?; let updated_entities = self.subscribed_entities.entities.read().clone().into_iter().collect(); diff --git a/crates/torii/client/src/client/storage.rs b/crates/torii/client/src/client/storage.rs index fd7221640b..ed89258806 100644 --- a/crates/torii/client/src/client/storage.rs +++ b/crates/torii/client/src/client/storage.rs @@ -203,13 +203,13 @@ mod tests { fn err_if_set_values_too_many() { let storage = create_dummy_storage(); let keys = vec![felt!("0x12345")]; - let entity = Query { + let query = Query { model: "Position".into(), clause: Clause::Keys(KeysClause { keys: keys.clone() }), }; let values = vec![felt!("1"), felt!("2"), felt!("3"), felt!("4"), felt!("5")]; - let model = cairo_short_string_to_felt(&entity.model).unwrap(); + let model = cairo_short_string_to_felt(&query.model).unwrap(); let result = storage.set_entity_storage(model, keys, values); assert!(storage.storage.read().is_empty()); @@ -223,13 +223,13 @@ mod tests { fn err_if_set_values_too_few() { let storage = create_dummy_storage(); let keys = vec![felt!("0x12345")]; - let entity = Query { + let query = Query { model: "Position".into(), clause: Clause::Keys(KeysClause { keys: keys.clone() }), }; let values = vec![felt!("1"), felt!("2")]; - let model = cairo_short_string_to_felt(&entity.model).unwrap(); + let model = cairo_short_string_to_felt(&query.model).unwrap(); let result = storage.set_entity_storage(model, keys, values); assert!(storage.storage.read().is_empty()); @@ -243,14 +243,14 @@ mod tests { fn set_and_get_entity_value() { let storage = create_dummy_storage(); let keys = vec![felt!("0x12345")]; - let entity = Query { + let query = Query { model: "Position".into(), clause: Clause::Keys(KeysClause { keys: keys.clone() }), }; assert!(storage.storage.read().is_empty(), "storage must be empty initially"); - let model = storage.metadata.read().model(&entity.model).cloned().unwrap(); + let model = storage.metadata.read().model(&query.model).cloned().unwrap(); let expected_storage_addresses = compute_all_storage_addresses( cairo_short_string_to_felt(&model.name).unwrap(), @@ -259,7 +259,7 @@ mod tests { ); let expected_values = vec![felt!("1"), felt!("2"), felt!("3"), felt!("4")]; - let model_name_in_felt = cairo_short_string_to_felt(&entity.model).unwrap(); + let model_name_in_felt = cairo_short_string_to_felt(&query.model).unwrap(); storage .set_entity_storage(model_name_in_felt, keys.clone(), expected_values.clone()) diff --git a/crates/torii/client/src/client/subscription.rs b/crates/torii/client/src/client/subscription.rs index 6c95a20f4f..1935f727d2 100644 --- a/crates/torii/client/src/client/subscription.rs +++ b/crates/torii/client/src/client/subscription.rs @@ -90,12 +90,12 @@ impl SubscribedEntities { Ok(()) } - pub(super) fn remove_entity(&self, entity: Query) -> Result<(), Error> { - if !self.entities.write().remove(&entity) { + pub(super) fn remove_entity(&self, query: Query) -> Result<(), Error> { + if !self.entities.write().remove(&query) { return Ok(()); } - let keys = if let Clause::Keys(clause) = entity.clause { + let keys = if let Clause::Keys(clause) = query.clause { clause.keys } else { return Err(Error::UnsupportedQuery); @@ -105,13 +105,12 @@ impl SubscribedEntities { .metadata .read() .models - .get(&entity.model) + .get(&query.model) .map(|c| c.packed_size) - .ok_or(Error::UnknownModel(entity.model.clone()))?; + .ok_or(Error::UnknownModel(query.model.clone()))?; let storage_addresses = compute_all_storage_addresses( - cairo_short_string_to_felt(&entity.model) - .map_err(ParseError::CairoShortStringToFelt)?, + cairo_short_string_to_felt(&query.model).map_err(ParseError::CairoShortStringToFelt)?, &keys, model_packed_size, ); @@ -206,11 +205,7 @@ impl SubscriptionService { let storage_entries = diff.storage_diffs.into_iter().find_map(|d| { let expected = self.world_metadata.read().world_address; let current = d.address; - if current == expected { - Some(d.storage_entries) - } else { - None - } + if current == expected { Some(d.storage_entries) } else { None } }); let Some(entries) = storage_entries else { @@ -300,26 +295,26 @@ mod tests { let metadata = self::create_dummy_metadata(); - let entity = Query { model: model_name, clause: Clause::Keys(KeysClause { keys }) }; + let query = Query { model: model_name, clause: Clause::Keys(KeysClause { keys }) }; let subscribed_entities = super::SubscribedEntities::new(Arc::new(RwLock::new(metadata))); - subscribed_entities.add_entities(vec![entity.clone()]).expect("able to add entity"); + subscribed_entities.add_entities(vec![query.clone()]).expect("able to add entity"); let actual_storage_addresses_count = subscribed_entities.subscribed_storage_addresses.read().len(); let actual_storage_addresses = subscribed_entities.subscribed_storage_addresses.read().clone(); - assert!(subscribed_entities.entities.read().contains(&entity)); + assert!(subscribed_entities.entities.read().contains(&query)); assert_eq!(actual_storage_addresses_count, expected_storage_addresses.len()); assert!(expected_storage_addresses.all(|addr| actual_storage_addresses.contains(&addr))); - subscribed_entities.remove_entities(vec![entity.clone()]).expect("able to remove entities"); + subscribed_entities.remove_entities(vec![query.clone()]).expect("able to remove entities"); let actual_storage_addresses_count_after = subscribed_entities.subscribed_storage_addresses.read().len(); assert_eq!(actual_storage_addresses_count_after, 0); - assert!(!subscribed_entities.entities.read().contains(&entity)); + assert!(!subscribed_entities.entities.read().contains(&query)); } } diff --git a/crates/torii/core/src/error.rs b/crates/torii/core/src/error.rs index 49da62fbcd..cdcf6b9b95 100644 --- a/crates/torii/core/src/error.rs +++ b/crates/torii/core/src/error.rs @@ -1,4 +1,3 @@ -use dojo_types::schema::QueryError; use starknet::core::types::{FromByteSliceError, FromStrError}; use starknet::core::utils::CairoShortStringToFeltError; diff --git a/crates/torii/grpc/src/server/mod.rs b/crates/torii/grpc/src/server/mod.rs index 6d426fbbce..7d226559a1 100644 --- a/crates/torii/grpc/src/server/mod.rs +++ b/crates/torii/grpc/src/server/mod.rs @@ -1,6 +1,5 @@ pub mod error; pub mod logger; -pub mod query; pub mod subscription; use std::future::Future; diff --git a/crates/torii/grpc/src/server/query.rs b/crates/torii/grpc/src/server/query.rs deleted file mode 100644 index 511cac2824..0000000000 --- a/crates/torii/grpc/src/server/query.rs +++ /dev/null @@ -1,64 +0,0 @@ -use serde::{Deserialize, Serialize}; -use starknet_crypto::FieldElement; - -#[derive(Debug, thiserror::Error)] -pub enum QueryError { - #[error("unsupported query")] - UnsupportedQuery, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] -pub struct Query { - pub model: String, - pub clause: Clause, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] -pub enum Clause { - Keys(KeysClause), - Attribute(AttributeClause), - Composite(CompositeClause), -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] -pub struct KeysClause { - pub keys: Vec, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] -pub struct AttributeClause { - pub attribute: String, - pub operator: ComparisonOperator, - pub value: Value, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] -pub struct CompositeClause { - pub operator: LogicalOperator, - pub clauses: Vec, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] -pub enum LogicalOperator { - And, - Or, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] -pub enum ComparisonOperator { - Eq, - Neq, - Gt, - Gte, - Lt, - Lte, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] -pub enum Value { - String(String), - Int(i64), - UInt(u64), - Bool(bool), - Bytes(Vec), -} diff --git a/crates/torii/grpc/src/server/subscription.rs b/crates/torii/grpc/src/server/subscription.rs index fe15c7577a..983451fec4 100644 --- a/crates/torii/grpc/src/server/subscription.rs +++ b/crates/torii/grpc/src/server/subscription.rs @@ -3,7 +3,6 @@ use std::future::Future; use std::sync::Arc; use std::task::Poll; -use dojo_types::schema::EntityQuery; use futures_util::future::BoxFuture; use futures_util::FutureExt; use rand::Rng; From 1a1eb33d80d96656011c189b370746b5218aa98e Mon Sep 17 00:00:00 2001 From: broody Date: Wed, 15 Nov 2023 12:24:51 -0800 Subject: [PATCH 06/16] Move model to individual query clause --- crates/torii/client/src/client/mod.rs | 47 ++++++++++--------- crates/torii/client/src/client/storage.rs | 24 ++-------- .../torii/client/src/client/subscription.rs | 27 ++++++----- crates/torii/grpc/proto/types.proto | 18 +++---- crates/torii/grpc/src/server/mod.rs | 18 +++++-- crates/torii/grpc/src/types.rs | 15 ++++-- 6 files changed, 78 insertions(+), 71 deletions(-) diff --git a/crates/torii/client/src/client/mod.rs b/crates/torii/client/src/client/mod.rs index d14ade458f..4b629fd43f 100644 --- a/crates/torii/client/src/client/mod.rs +++ b/crates/torii/client/src/client/mod.rs @@ -67,18 +67,18 @@ impl Client { // TODO: change this to querying the gRPC url instead let subbed_entities = subbed_entities.entities.read().clone(); - for Query { model, clause } in subbed_entities { - let model_reader = world_reader.model(&model).await?; - let keys = if let Clause::Keys(clause) = clause { - clause.keys + for Query { clause } in subbed_entities { + let clause = if let Clause::Keys(clause) = clause { + clause } else { return Err(Error::UnsupportedQuery); }; - let values = model_reader.entity_storage(&keys).await?; + let model_reader = world_reader.model(&clause.model).await?; + let values = model_reader.entity_storage(&clause.keys).await?; client_storage.set_entity_storage( - cairo_short_string_to_felt(&model).unwrap(), - keys, + cairo_short_string_to_felt(&clause.model).unwrap(), + clause.keys, values, )?; } @@ -111,25 +111,26 @@ impl Client { /// If the requested entity is not among the synced entities, it will attempt to fetch it from /// the RPC. pub async fn entity(&self, query: &Query) -> Result, Error> { - let Some(mut schema) = self.metadata.read().model(&query.model).map(|m| m.schema.clone()) - else { - return Ok(None); - }; - - let keys = if let Clause::Keys(clause) = query.clone().clause { - clause.keys + let clause = if let Clause::Keys(clause) = query.clone().clause { + clause } else { return Err(Error::UnsupportedQuery); }; + let Some(mut schema) = self.metadata.read().model(&clause.model).map(|m| m.schema.clone()) + else { + return Ok(None); + }; + if !self.subscribed_entities.is_synced(query) { - let model = self.world_reader.model(&query.model).await?; - return Ok(Some(model.entity(&keys).await?)); + let model = self.world_reader.model(&clause.model).await?; + return Ok(Some(model.entity(&clause.keys).await?)); } let Ok(Some(raw_values)) = self.storage.get_entity_storage( - cairo_short_string_to_felt(&query.model).map_err(ParseError::CairoShortStringToFelt)?, - &keys, + cairo_short_string_to_felt(&clause.model) + .map_err(ParseError::CairoShortStringToFelt)?, + &clause.keys, ) else { return Ok(Some(schema)); }; @@ -137,12 +138,12 @@ impl Client { let layout = self .metadata .read() - .model(&query.model) + .model(&clause.model) .map(|m| m.layout.clone()) .expect("qed; layout should exist"); let unpacked = unpack(raw_values, layout).unwrap(); - let mut keys_and_unpacked = [keys.to_vec(), unpacked].concat(); + let mut keys_and_unpacked = [clause.keys.to_vec(), unpacked].concat(); schema.deserialize(&mut keys_and_unpacked).unwrap(); @@ -171,13 +172,13 @@ impl Client { /// NOTE: This will establish a new subscription stream with the server. pub async fn add_entities_to_sync(&self, queries: Vec) -> Result<(), Error> { for query in &queries { - let keys = if let Clause::Keys(clause) = query.clone().clause { - clause.keys + let clause = if let Clause::Keys(clause) = query.clone().clause { + clause } else { return Err(Error::UnsupportedQuery); }; - self.initiate_entity(&query.model, keys.clone()).await?; + self.initiate_entity(&clause.model, clause.keys.clone()).await?; } self.subscribed_entities.add_entities(queries)?; diff --git a/crates/torii/client/src/client/storage.rs b/crates/torii/client/src/client/storage.rs index ed89258806..d56b7db506 100644 --- a/crates/torii/client/src/client/storage.rs +++ b/crates/torii/client/src/client/storage.rs @@ -173,7 +173,6 @@ mod tests { use parking_lot::RwLock; use starknet::core::utils::cairo_short_string_to_felt; use starknet::macros::felt; - use torii_grpc::types::{Clause, KeysClause, Query}; use crate::client::error::Error; use crate::utils::compute_all_storage_addresses; @@ -203,13 +202,8 @@ mod tests { fn err_if_set_values_too_many() { let storage = create_dummy_storage(); let keys = vec![felt!("0x12345")]; - let query = Query { - model: "Position".into(), - clause: Clause::Keys(KeysClause { keys: keys.clone() }), - }; - let values = vec![felt!("1"), felt!("2"), felt!("3"), felt!("4"), felt!("5")]; - let model = cairo_short_string_to_felt(&query.model).unwrap(); + let model = cairo_short_string_to_felt("Position").unwrap(); let result = storage.set_entity_storage(model, keys, values); assert!(storage.storage.read().is_empty()); @@ -223,13 +217,8 @@ mod tests { fn err_if_set_values_too_few() { let storage = create_dummy_storage(); let keys = vec![felt!("0x12345")]; - let query = Query { - model: "Position".into(), - clause: Clause::Keys(KeysClause { keys: keys.clone() }), - }; - let values = vec![felt!("1"), felt!("2")]; - let model = cairo_short_string_to_felt(&query.model).unwrap(); + let model = cairo_short_string_to_felt("Position").unwrap(); let result = storage.set_entity_storage(model, keys, values); assert!(storage.storage.read().is_empty()); @@ -243,15 +232,10 @@ mod tests { fn set_and_get_entity_value() { let storage = create_dummy_storage(); let keys = vec![felt!("0x12345")]; - let query = Query { - model: "Position".into(), - clause: Clause::Keys(KeysClause { keys: keys.clone() }), - }; assert!(storage.storage.read().is_empty(), "storage must be empty initially"); - let model = storage.metadata.read().model(&query.model).cloned().unwrap(); - + let model = storage.metadata.read().model("Position").cloned().unwrap(); let expected_storage_addresses = compute_all_storage_addresses( cairo_short_string_to_felt(&model.name).unwrap(), &keys, @@ -259,7 +243,7 @@ mod tests { ); let expected_values = vec![felt!("1"), felt!("2"), felt!("3"), felt!("4")]; - let model_name_in_felt = cairo_short_string_to_felt(&query.model).unwrap(); + let model_name_in_felt = cairo_short_string_to_felt("Position").unwrap(); storage .set_entity_storage(model_name_in_felt, keys.clone(), expected_values.clone()) diff --git a/crates/torii/client/src/client/subscription.rs b/crates/torii/client/src/client/subscription.rs index 1935f727d2..ae53f931eb 100644 --- a/crates/torii/client/src/client/subscription.rs +++ b/crates/torii/client/src/client/subscription.rs @@ -61,8 +61,8 @@ impl SubscribedEntities { return Ok(()); } - let keys = if let Clause::Keys(clause) = entity.clause { - clause.keys + let clause = if let Clause::Keys(clause) = entity.clause { + clause } else { return Err(Error::UnsupportedQuery); }; @@ -71,14 +71,14 @@ impl SubscribedEntities { .metadata .read() .models - .get(&entity.model) + .get(&clause.model) .map(|c| c.packed_size) - .ok_or(Error::UnknownModel(entity.model.clone()))?; + .ok_or(Error::UnknownModel(clause.model.clone()))?; let storage_addresses = compute_all_storage_addresses( - cairo_short_string_to_felt(&entity.model) + cairo_short_string_to_felt(&clause.model) .map_err(ParseError::CairoShortStringToFelt)?, - &keys, + &clause.keys, model_packed_size, ); @@ -95,8 +95,8 @@ impl SubscribedEntities { return Ok(()); } - let keys = if let Clause::Keys(clause) = query.clause { - clause.keys + let clause = if let Clause::Keys(clause) = query.clause { + clause } else { return Err(Error::UnsupportedQuery); }; @@ -105,13 +105,14 @@ impl SubscribedEntities { .metadata .read() .models - .get(&query.model) + .get(&clause.model) .map(|c| c.packed_size) - .ok_or(Error::UnknownModel(query.model.clone()))?; + .ok_or(Error::UnknownModel(clause.model.clone()))?; let storage_addresses = compute_all_storage_addresses( - cairo_short_string_to_felt(&query.model).map_err(ParseError::CairoShortStringToFelt)?, - &keys, + cairo_short_string_to_felt(&clause.model) + .map_err(ParseError::CairoShortStringToFelt)?, + &clause.keys, model_packed_size, ); @@ -295,7 +296,7 @@ mod tests { let metadata = self::create_dummy_metadata(); - let query = Query { model: model_name, clause: Clause::Keys(KeysClause { keys }) }; + let query = Query { clause: Clause::Keys(KeysClause { model: model_name, keys }) }; let subscribed_entities = super::SubscribedEntities::new(Arc::new(RwLock::new(metadata))); subscribed_entities.add_entities(vec![query.clone()]).expect("able to add entity"); diff --git a/crates/torii/grpc/proto/types.proto b/crates/torii/grpc/proto/types.proto index eb2e6e9fad..1c853c005b 100644 --- a/crates/torii/grpc/proto/types.proto +++ b/crates/torii/grpc/proto/types.proto @@ -54,8 +54,7 @@ message EntityUpdate { } message EntityQuery { - string model = 1; - Clause clause = 2; + Clause clause = 1; } message Clause { @@ -67,18 +66,21 @@ message Clause { } message KeysClause { - repeated bytes keys = 1; + string model = 1; + repeated bytes keys = 2; } message AttributeClause { - string attribute = 1; - ComparisonOperator operator = 2; - Value value = 3; + string model = 1; + string attribute = 2; + ComparisonOperator operator = 3; + Value value = 4; } message CompositeClause { - LogicalOperator operator = 1; - repeated Clause clauses = 2; + string model = 1; + LogicalOperator operator = 2; + repeated Clause clauses = 3; } enum LogicalOperator { diff --git a/crates/torii/grpc/src/server/mod.rs b/crates/torii/grpc/src/server/mod.rs index 7d226559a1..4c32b4ce7a 100644 --- a/crates/torii/grpc/src/server/mod.rs +++ b/crates/torii/grpc/src/server/mod.rs @@ -21,10 +21,12 @@ use tokio::sync::mpsc::Receiver; use tokio_stream::wrappers::{ReceiverStream, TcpListenerStream}; use tonic::transport::Server; use tonic::{Request, Response, Status}; -use torii_core::error::{Error, ParseError}; +use torii_core::error::{Error, ParseError, QueryError}; use torii_core::model::{parse_sql_model_members, SqlModelMember}; use self::subscription::SubscribeRequest; +use crate::proto::types::clause::ClauseType; +use crate::proto::types::KeysClause; use crate::proto::world::world_server::WorldServer; use crate::proto::{self}; @@ -144,11 +146,21 @@ impl DojoWorld { { let mut subs = Vec::with_capacity(queries.len()); for query in queries { - let model = cairo_short_string_to_felt(&query.model) + let clause: KeysClause = query + .clone() + .clause + .ok_or(QueryError::UnsupportedQuery) + .and_then(|clause| clause.clause_type.ok_or(QueryError::UnsupportedQuery)) + .and_then(|clause_type| match clause_type { + ClauseType::Keys(clause) => Ok(clause), + _ => Err(QueryError::UnsupportedQuery), + })?; + + let model = cairo_short_string_to_felt(&clause.model) .map_err(ParseError::CairoShortStringToFelt)?; let proto::types::ModelMetadata { packed_size, .. } = - self.model_metadata(&query.model).await?; + self.model_metadata(&clause.model).await?; subs.push(SubscribeRequest { query, diff --git a/crates/torii/grpc/src/types.rs b/crates/torii/grpc/src/types.rs index f98acd07ab..e62dc3b8a8 100644 --- a/crates/torii/grpc/src/types.rs +++ b/crates/torii/grpc/src/types.rs @@ -12,7 +12,6 @@ use crate::proto; #[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] pub struct Query { - pub model: String, pub clause: Clause, } @@ -25,11 +24,13 @@ pub enum Clause { #[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] pub struct KeysClause { + pub model: String, pub keys: Vec, } #[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] pub struct AttributeClause { + pub model: String, pub attribute: String, pub operator: ComparisonOperator, pub value: Value, @@ -37,6 +38,7 @@ pub struct AttributeClause { #[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] pub struct CompositeClause { + pub model: String, pub operator: LogicalOperator, pub clauses: Vec, } @@ -103,7 +105,7 @@ impl TryFrom for dojo_types::WorldMetadata { impl From for proto::types::EntityQuery { fn from(value: Query) -> Self { - Self { model: value.model, clause: Some(value.clause.into()) } + Self { clause: Some(value.clause.into()) } } } @@ -125,7 +127,10 @@ impl From for proto::types::Clause { impl From for proto::types::KeysClause { fn from(value: KeysClause) -> Self { - Self { keys: value.keys.iter().map(|k| k.to_bytes_be().into()).collect() } + Self { + model: value.model, + keys: value.keys.iter().map(|k| k.to_bytes_be().into()).collect(), + } } } @@ -139,13 +144,14 @@ impl TryFrom for KeysClause { .map(|k| FieldElement::from_byte_slice_be(&k)) .collect::, _>>()?; - Ok(Self { keys }) + Ok(Self { model: value.model, keys }) } } impl From for proto::types::AttributeClause { fn from(value: AttributeClause) -> Self { Self { + model: value.model, attribute: value.attribute, operator: value.operator as i32, value: Some(value.value.into()), @@ -156,6 +162,7 @@ impl From for proto::types::AttributeClause { impl From for proto::types::CompositeClause { fn from(value: CompositeClause) -> Self { Self { + model: value.model, operator: value.operator as i32, clauses: value.clauses.into_iter().map(|clause| clause.into()).collect(), } From 5e984bb389d76969a962d39bc6084c6b56a572ec Mon Sep 17 00:00:00 2001 From: broody Date: Wed, 15 Nov 2023 13:47:14 -0800 Subject: [PATCH 07/16] Refactor subscription argument to keys --- crates/torii/client/src/client/mod.rs | 88 ++++++++----------- .../torii/client/src/client/subscription.rs | 72 ++++++--------- crates/torii/grpc/proto/world.proto | 4 +- crates/torii/grpc/src/client.rs | 6 +- crates/torii/grpc/src/server/mod.rs | 34 +++---- crates/torii/grpc/src/server/subscription.rs | 21 ++--- 6 files changed, 88 insertions(+), 137 deletions(-) diff --git a/crates/torii/client/src/client/mod.rs b/crates/torii/client/src/client/mod.rs index 4b629fd43f..ba713b5c5a 100644 --- a/crates/torii/client/src/client/mod.rs +++ b/crates/torii/client/src/client/mod.rs @@ -17,7 +17,7 @@ use starknet::providers::JsonRpcClient; use starknet_crypto::FieldElement; use tokio::sync::RwLock as AsyncRwLock; use torii_grpc::client::EntityUpdateStreaming; -use torii_grpc::types::{Clause, Query}; +use torii_grpc::types::KeysClause; use self::error::{Error, ParseError}; use self::storage::ModelStorage; @@ -47,7 +47,7 @@ impl Client { torii_url: String, rpc_url: String, world: FieldElement, - queries: Option>, + entities_keys: Option>, ) -> Result { let mut grpc_client = torii_grpc::client::WorldClient::new(torii_url, world).await?; @@ -62,23 +62,18 @@ impl Client { let provider = JsonRpcClient::new(HttpTransport::new(rpc_url)); let world_reader = WorldContractReader::new(world, provider); - if let Some(queries) = queries { - subbed_entities.add_entities(queries)?; + if let Some(keys) = entities_keys { + subbed_entities.add_entities(keys)?; // TODO: change this to querying the gRPC url instead - let subbed_entities = subbed_entities.entities.read().clone(); - for Query { clause } in subbed_entities { - let clause = if let Clause::Keys(clause) = clause { - clause - } else { - return Err(Error::UnsupportedQuery); - }; - let model_reader = world_reader.model(&clause.model).await?; - let values = model_reader.entity_storage(&clause.keys).await?; + let subbed_entities = subbed_entities.entities_keys.read().clone(); + for keys in subbed_entities { + let model_reader = world_reader.model(&keys.model).await?; + let values = model_reader.entity_storage(&keys.keys).await?; client_storage.set_entity_storage( - cairo_short_string_to_felt(&clause.model).unwrap(), - clause.keys, + cairo_short_string_to_felt(&keys.model).unwrap(), + keys.keys, values, )?; } @@ -99,8 +94,8 @@ impl Client { self.metadata.read() } - pub fn subscribed_entities(&self) -> RwLockReadGuard<'_, HashSet> { - self.subscribed_entities.entities.read() + pub fn subscribed_entities(&self) -> RwLockReadGuard<'_, HashSet> { + self.subscribed_entities.entities_keys.read() } /// Returns the model value of an entity. @@ -110,27 +105,20 @@ impl Client { /// /// If the requested entity is not among the synced entities, it will attempt to fetch it from /// the RPC. - pub async fn entity(&self, query: &Query) -> Result, Error> { - let clause = if let Clause::Keys(clause) = query.clone().clause { - clause - } else { - return Err(Error::UnsupportedQuery); - }; - - let Some(mut schema) = self.metadata.read().model(&clause.model).map(|m| m.schema.clone()) + pub async fn entity(&self, keys: &KeysClause) -> Result, Error> { + let Some(mut schema) = self.metadata.read().model(&keys.model).map(|m| m.schema.clone()) else { return Ok(None); }; - if !self.subscribed_entities.is_synced(query) { - let model = self.world_reader.model(&clause.model).await?; - return Ok(Some(model.entity(&clause.keys).await?)); + if !self.subscribed_entities.is_synced(keys) { + let model = self.world_reader.model(&keys.model).await?; + return Ok(Some(model.entity(&keys.keys).await?)); } let Ok(Some(raw_values)) = self.storage.get_entity_storage( - cairo_short_string_to_felt(&clause.model) - .map_err(ParseError::CairoShortStringToFelt)?, - &clause.keys, + cairo_short_string_to_felt(&keys.model).map_err(ParseError::CairoShortStringToFelt)?, + &keys.keys, ) else { return Ok(Some(schema)); }; @@ -138,12 +126,12 @@ impl Client { let layout = self .metadata .read() - .model(&clause.model) + .model(&keys.model) .map(|m| m.layout.clone()) .expect("qed; layout should exist"); let unpacked = unpack(raw_values, layout).unwrap(); - let mut keys_and_unpacked = [clause.keys.to_vec(), unpacked].concat(); + let mut keys_and_unpacked = [keys.keys.to_vec(), unpacked].concat(); schema.deserialize(&mut keys_and_unpacked).unwrap(); @@ -153,8 +141,9 @@ impl Client { /// Initiate the entity subscriptions and returns a [SubscriptionService] which when await'ed /// will execute the subscription service and starts the syncing process. pub async fn start_subscription(&self) -> Result { - let entities = self.subscribed_entities.entities.read().clone().into_iter().collect(); - let sub_res_stream = self.initiate_subscription(entities).await?; + let entities_keys = + self.subscribed_entities.entities_keys.read().clone().into_iter().collect(); + let sub_res_stream = self.initiate_subscription(entities_keys).await?; let (service, handle) = SubscriptionService::new( Arc::clone(&self.storage), @@ -170,21 +159,15 @@ impl Client { /// Adds entities to the list of entities to be synced. /// /// NOTE: This will establish a new subscription stream with the server. - pub async fn add_entities_to_sync(&self, queries: Vec) -> Result<(), Error> { - for query in &queries { - let clause = if let Clause::Keys(clause) = query.clone().clause { - clause - } else { - return Err(Error::UnsupportedQuery); - }; - - self.initiate_entity(&clause.model, clause.keys.clone()).await?; + pub async fn add_entities_to_sync(&self, entities_keys: Vec) -> Result<(), Error> { + for keys in &entities_keys { + self.initiate_entity(&keys.model, keys.keys.clone()).await?; } - self.subscribed_entities.add_entities(queries)?; + self.subscribed_entities.add_entities(entities_keys)?; let updated_entities = - self.subscribed_entities.entities.read().clone().into_iter().collect(); + self.subscribed_entities.entities_keys.read().clone().into_iter().collect(); let sub_res_stream = self.initiate_subscription(updated_entities).await?; match self.sub_client_handle.get() { @@ -197,11 +180,14 @@ impl Client { /// Removes entities from the list of entities to be synced. /// /// NOTE: This will establish a new subscription stream with the server. - pub async fn remove_entities_to_sync(&self, queries: Vec) -> Result<(), Error> { - self.subscribed_entities.remove_entities(queries)?; + pub async fn remove_entities_to_sync( + &self, + entities_keys: Vec, + ) -> Result<(), Error> { + self.subscribed_entities.remove_entities(entities_keys)?; let updated_entities = - self.subscribed_entities.entities.read().clone().into_iter().collect(); + self.subscribed_entities.entities_keys.read().clone().into_iter().collect(); let sub_res_stream = self.initiate_subscription(updated_entities).await?; match self.sub_client_handle.get() { @@ -217,10 +203,10 @@ impl Client { async fn initiate_subscription( &self, - queries: Vec, + keys: Vec, ) -> Result { let mut grpc_client = self.inner.write().await; - let stream = grpc_client.subscribe_entities(queries).await?; + let stream = grpc_client.subscribe_entities(keys).await?; Ok(stream) } diff --git a/crates/torii/client/src/client/subscription.rs b/crates/torii/client/src/client/subscription.rs index ae53f931eb..898d008130 100644 --- a/crates/torii/client/src/client/subscription.rs +++ b/crates/torii/client/src/client/subscription.rs @@ -12,7 +12,7 @@ use starknet::core::types::{StateDiff, StateUpdate}; use starknet::core::utils::cairo_short_string_to_felt; use starknet_crypto::FieldElement; use torii_grpc::client::EntityUpdateStreaming; -use torii_grpc::types::{Clause, Query}; +use torii_grpc::types::KeysClause; use super::error::{Error, ParseError}; use super::ModelStorage; @@ -24,61 +24,54 @@ pub enum SubscriptionEvent { pub struct SubscribedEntities { metadata: Arc>, - pub(super) entities: RwLock>, + pub(super) entities_keys: RwLock>, /// All the relevant storage addresses derived from the subscribed entities pub(super) subscribed_storage_addresses: RwLock>, } impl SubscribedEntities { - pub(super) fn is_synced(&self, entity: &Query) -> bool { - self.entities.read().contains(entity) + pub(super) fn is_synced(&self, keys: &KeysClause) -> bool { + self.entities_keys.read().contains(keys) } pub(super) fn new(metadata: Arc>) -> Self { Self { metadata, - entities: Default::default(), + entities_keys: Default::default(), subscribed_storage_addresses: Default::default(), } } - pub(super) fn add_entities(&self, entities: Vec) -> Result<(), Error> { - for entity in entities { - Self::add_entity(self, entity)?; + pub(super) fn add_entities(&self, entities_keys: Vec) -> Result<(), Error> { + for keys in entities_keys { + Self::add_entity(self, keys)?; } Ok(()) } - pub(super) fn remove_entities(&self, entities: Vec) -> Result<(), Error> { - for entity in entities { - Self::remove_entity(self, entity)?; + pub(super) fn remove_entities(&self, entities_keys: Vec) -> Result<(), Error> { + for keys in entities_keys { + Self::remove_entity(self, keys)?; } Ok(()) } - pub(super) fn add_entity(&self, entity: Query) -> Result<(), Error> { - if !self.entities.write().insert(entity.clone()) { + pub(super) fn add_entity(&self, keys: KeysClause) -> Result<(), Error> { + if !self.entities_keys.write().insert(keys.clone()) { return Ok(()); } - let clause = if let Clause::Keys(clause) = entity.clause { - clause - } else { - return Err(Error::UnsupportedQuery); - }; - let model_packed_size = self .metadata .read() .models - .get(&clause.model) + .get(&keys.model) .map(|c| c.packed_size) - .ok_or(Error::UnknownModel(clause.model.clone()))?; + .ok_or(Error::UnknownModel(keys.model.clone()))?; let storage_addresses = compute_all_storage_addresses( - cairo_short_string_to_felt(&clause.model) - .map_err(ParseError::CairoShortStringToFelt)?, - &clause.keys, + cairo_short_string_to_felt(&keys.model).map_err(ParseError::CairoShortStringToFelt)?, + &keys.keys, model_packed_size, ); @@ -90,29 +83,22 @@ impl SubscribedEntities { Ok(()) } - pub(super) fn remove_entity(&self, query: Query) -> Result<(), Error> { - if !self.entities.write().remove(&query) { + pub(super) fn remove_entity(&self, keys: KeysClause) -> Result<(), Error> { + if !self.entities_keys.write().remove(&keys) { return Ok(()); } - let clause = if let Clause::Keys(clause) = query.clause { - clause - } else { - return Err(Error::UnsupportedQuery); - }; - let model_packed_size = self .metadata .read() .models - .get(&clause.model) + .get(&keys.model) .map(|c| c.packed_size) - .ok_or(Error::UnknownModel(clause.model.clone()))?; + .ok_or(Error::UnknownModel(keys.model.clone()))?; let storage_addresses = compute_all_storage_addresses( - cairo_short_string_to_felt(&clause.model) - .map_err(ParseError::CairoShortStringToFelt)?, - &clause.keys, + cairo_short_string_to_felt(&keys.model).map_err(ParseError::CairoShortStringToFelt)?, + &keys.keys, model_packed_size, ); @@ -261,7 +247,7 @@ mod tests { use parking_lot::RwLock; use starknet::core::utils::cairo_short_string_to_felt; use starknet::macros::felt; - use torii_grpc::types::{Clause, KeysClause, Query}; + use torii_grpc::types::KeysClause; use crate::utils::compute_all_storage_addresses; @@ -296,26 +282,26 @@ mod tests { let metadata = self::create_dummy_metadata(); - let query = Query { clause: Clause::Keys(KeysClause { model: model_name, keys }) }; + let keys = KeysClause { model: model_name, keys }; let subscribed_entities = super::SubscribedEntities::new(Arc::new(RwLock::new(metadata))); - subscribed_entities.add_entities(vec![query.clone()]).expect("able to add entity"); + subscribed_entities.add_entities(vec![keys.clone()]).expect("able to add entity"); let actual_storage_addresses_count = subscribed_entities.subscribed_storage_addresses.read().len(); let actual_storage_addresses = subscribed_entities.subscribed_storage_addresses.read().clone(); - assert!(subscribed_entities.entities.read().contains(&query)); + assert!(subscribed_entities.entities_keys.read().contains(&keys)); assert_eq!(actual_storage_addresses_count, expected_storage_addresses.len()); assert!(expected_storage_addresses.all(|addr| actual_storage_addresses.contains(&addr))); - subscribed_entities.remove_entities(vec![query.clone()]).expect("able to remove entities"); + subscribed_entities.remove_entities(vec![keys.clone()]).expect("able to remove entities"); let actual_storage_addresses_count_after = subscribed_entities.subscribed_storage_addresses.read().len(); assert_eq!(actual_storage_addresses_count_after, 0); - assert!(!subscribed_entities.entities.read().contains(&query)); + assert!(!subscribed_entities.entities_keys.read().contains(&keys)); } } diff --git a/crates/torii/grpc/proto/world.proto b/crates/torii/grpc/proto/world.proto index 10734e7e8e..66f5fce09d 100644 --- a/crates/torii/grpc/proto/world.proto +++ b/crates/torii/grpc/proto/world.proto @@ -25,8 +25,8 @@ message MetadataResponse { } message SubscribeEntitiesRequest { - // The list of entity queries to subscribe to. - repeated types.EntityQuery queries = 1; + // The list of entity keys to subscribe to. + repeated types.KeysClause entities_keys = 1; } message SubscribeEntitiesResponse { diff --git a/crates/torii/grpc/src/client.rs b/crates/torii/grpc/src/client.rs index 9cf15c0e28..e4097849c3 100644 --- a/crates/torii/grpc/src/client.rs +++ b/crates/torii/grpc/src/client.rs @@ -7,7 +7,7 @@ use starknet_crypto::FieldElement; use crate::proto::world::{MetadataRequest, SubscribeEntitiesResponse}; use crate::proto::{self}; -use crate::types::Query; +use crate::types::KeysClause; #[derive(Debug, thiserror::Error)] pub enum Error { @@ -67,12 +67,12 @@ impl WorldClient { /// Subscribe to the state diff for a set of entities of a World. pub async fn subscribe_entities( &mut self, - queries: Vec, + entities_keys: Vec, ) -> Result { let stream = self .inner .subscribe_entities(SubscribeEntitiesRequest { - queries: queries.into_iter().map(|e| e.into()).collect(), + entities_keys: entities_keys.into_iter().map(|e| e.into()).collect(), }) .await .map_err(Error::Grpc) diff --git a/crates/torii/grpc/src/server/mod.rs b/crates/torii/grpc/src/server/mod.rs index 4c32b4ce7a..d5ccac1680 100644 --- a/crates/torii/grpc/src/server/mod.rs +++ b/crates/torii/grpc/src/server/mod.rs @@ -21,12 +21,10 @@ use tokio::sync::mpsc::Receiver; use tokio_stream::wrappers::{ReceiverStream, TcpListenerStream}; use tonic::transport::Server; use tonic::{Request, Response, Status}; -use torii_core::error::{Error, ParseError, QueryError}; +use torii_core::error::{Error, ParseError}; use torii_core::model::{parse_sql_model_members, SqlModelMember}; use self::subscription::SubscribeRequest; -use crate::proto::types::clause::ClauseType; -use crate::proto::types::KeysClause; use crate::proto::world::world_server::WorldServer; use crate::proto::{self}; @@ -141,29 +139,19 @@ impl DojoWorld { async fn subscribe_entities( &self, - queries: Vec, + entities_keys: Vec, ) -> Result>, Error> { - let mut subs = Vec::with_capacity(queries.len()); - for query in queries { - let clause: KeysClause = query - .clone() - .clause - .ok_or(QueryError::UnsupportedQuery) - .and_then(|clause| clause.clause_type.ok_or(QueryError::UnsupportedQuery)) - .and_then(|clause_type| match clause_type { - ClauseType::Keys(clause) => Ok(clause), - _ => Err(QueryError::UnsupportedQuery), - })?; - - let model = cairo_short_string_to_felt(&clause.model) + let mut subs = Vec::with_capacity(entities_keys.len()); + for keys in entities_keys { + let model = cairo_short_string_to_felt(&keys.model) .map_err(ParseError::CairoShortStringToFelt)?; let proto::types::ModelMetadata { packed_size, .. } = - self.model_metadata(&clause.model).await?; + self.model_metadata(&keys.model).await?; subs.push(SubscribeRequest { - query, + keys, model: subscription::ModelMetadata { name: model, packed_size: packed_size as usize, @@ -199,9 +187,11 @@ impl proto::world::world_server::World for DojoWorld { &self, request: Request, ) -> ServiceResult { - let SubscribeEntitiesRequest { queries } = request.into_inner(); - let rx = - self.subscribe_entities(queries).await.map_err(|e| Status::internal(e.to_string()))?; + let SubscribeEntitiesRequest { entities_keys } = request.into_inner(); + let rx = self + .subscribe_entities(entities_keys) + .await + .map_err(|e| Status::internal(e.to_string()))?; Ok(Response::new(Box::pin(ReceiverStream::new(rx)) as Self::SubscribeEntitiesStream)) } } diff --git a/crates/torii/grpc/src/server/subscription.rs b/crates/torii/grpc/src/server/subscription.rs index 983451fec4..bc20140df7 100644 --- a/crates/torii/grpc/src/server/subscription.rs +++ b/crates/torii/grpc/src/server/subscription.rs @@ -15,12 +15,11 @@ use starknet::providers::Provider; use starknet_crypto::{poseidon_hash_many, FieldElement}; use tokio::sync::mpsc::{channel, Receiver, Sender}; use tokio::sync::RwLock; -use torii_core::error::{Error, ParseError, QueryError}; +use torii_core::error::{Error, ParseError}; use tracing::{debug, error, trace}; use super::error::SubscriptionError; use crate::proto; -use crate::proto::types::EntityQuery; use crate::types::KeysClause; pub struct ModelMetadata { @@ -30,7 +29,7 @@ pub struct ModelMetadata { pub struct SubscribeRequest { pub model: ModelMetadata, - pub query: EntityQuery, + pub keys: proto::types::KeysClause, } impl SubscribeRequest { @@ -74,23 +73,13 @@ impl SubscriberManager { let storage_addresses = reqs .into_iter() .map(|req| { - let clause: KeysClause = req - .query - .clause - .ok_or(QueryError::UnsupportedQuery) - .and_then(|clause| clause.clause_type.ok_or(QueryError::UnsupportedQuery)) - .and_then(|clause_type| match clause_type { - proto::types::clause::ClauseType::Keys(clause) => Ok(clause), - _ => Err(QueryError::UnsupportedQuery), - }) - .map_err(Error::QueryError)? - .try_into() - .map_err(ParseError::FromByteSliceError)?; + let keys: KeysClause = + req.keys.try_into().map_err(ParseError::FromByteSliceError)?; let base = poseidon_hash_many(&[ short_string!("dojo_storage"), req.model.name, - poseidon_hash_many(&clause.keys), + poseidon_hash_many(&keys.keys), ]); let res = (0..req.model.packed_size) From 0da655014b542c6dfa1d5f1a45526a2a941e2607 Mon Sep 17 00:00:00 2001 From: broody Date: Wed, 15 Nov 2023 14:43:02 -0800 Subject: [PATCH 08/16] Add Torii grpc retrieve entities endpoint --- Cargo.lock | 1 + crates/dojo-types/src/primitive.rs | 171 ++++++++++------------- crates/dojo-types/src/schema.rs | 14 ++ crates/torii/core/Cargo.toml | 1 + crates/torii/core/src/error.rs | 12 ++ crates/torii/core/src/model.rs | 203 +++++++++++++++++++++++++++- crates/torii/grpc/proto/types.proto | 26 +++- crates/torii/grpc/proto/world.proto | 14 +- crates/torii/grpc/src/server/mod.rs | 82 ++++++++++- crates/torii/grpc/src/types.rs | 4 +- scripts/rust_fmt.sh | 2 +- 11 files changed, 421 insertions(+), 109 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c7d80ecd4a..c86692ab43 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8300,6 +8300,7 @@ dependencies = [ "base64 0.21.5", "camino", "chrono", + "crypto-bigint", "dojo-test-utils", "dojo-types", "dojo-world", diff --git a/crates/dojo-types/src/primitive.rs b/crates/dojo-types/src/primitive.rs index d5a159bad9..dbb266df86 100644 --- a/crates/dojo-types/src/primitive.rs +++ b/crates/dojo-types/src/primitive.rs @@ -33,6 +33,8 @@ pub enum PrimitiveError { NotEnoughFieldElements, #[error("Unsupported CairoType for SQL formatting")] UnsupportedType, + #[error("Set value type mismatch")] + TypeMismatch, #[error(transparent)] ValueOutOfRange(#[from] ValueOutOfRangeError), } @@ -44,97 +46,59 @@ pub enum SqlType { Text, } -impl Primitive { - /// If the `Primitive` is a u8, returns the associated [`u8`]. Returns `None` otherwise. - pub fn as_u8(&self) -> Option { - match self { - Primitive::U8(value) => *value, - _ => None, - } - } - - /// If the `Primitive` is a u16, returns the associated [`u16`]. Returns `None` otherwise. - pub fn as_u16(&self) -> Option { - match self { - Primitive::U16(value) => *value, - _ => None, - } - } - - /// If the `Primitive` is a u32, returns the associated [`u32`]. Returns `None` otherwise. - pub fn as_u32(&self) -> Option { - match self { - Primitive::U32(value) => *value, - _ => None, - } - } - - /// If the `Primitive` is a u64, returns the associated [`u64`]. Returns `None` otherwise. - pub fn as_u64(&self) -> Option { - match self { - Primitive::U64(value) => *value, - _ => None, - } - } - - /// If the `Primitive` is a u128, returns the associated [`u128`]. Returns `None` otherwise. - pub fn as_u128(&self) -> Option { - match self { - Primitive::U128(value) => *value, - _ => None, - } - } - - /// If the `Primitive` is a u256, returns the associated [`U256`]. Returns `None` otherwise. - pub fn as_u256(&self) -> Option { - match self { - Primitive::U256(value) => *value, - _ => None, - } - } - - /// If the `Primitive` is a felt252, returns the associated [`FieldElement`]. Returns `None` - /// otherwise. - pub fn as_felt252(&self) -> Option { - match self { - Primitive::Felt252(value) => *value, - _ => None, - } - } - - /// If the `Primitive` is a ClassHash, returns the associated [`FieldElement`]. Returns `None` - /// otherwise. - pub fn as_class_hash(&self) -> Option { - match self { - Primitive::ClassHash(value) => *value, - _ => None, +/// Macro to generate setter methods for Primitive enum variants. +macro_rules! set_primitive { + ($method_name:ident, $variant:ident, $type:ty) => { + /// Sets the inner value of the `Primitive` enum if variant matches. + pub fn $method_name(&mut self, value: Option<$type>) -> Result<(), PrimitiveError> { + match self { + Primitive::$variant(_) => { + *self = Primitive::$variant(value); + Ok(()) + } + _ => Err(PrimitiveError::TypeMismatch), + } } - } + }; +} - /// If the `Primitive` is a ContractAddress, returns the associated [`FieldElement`]. Returns - /// `None` otherwise. - pub fn as_contract_address(&self) -> Option { - match self { - Primitive::ContractAddress(value) => *value, - _ => None, +/// Macro to generate getter methods for Primitive enum variants. +macro_rules! get_primitive { + ($method_name:ident, $variant:ident, $type:ty) => { + /// If the `Primitive` is type T, returns the associated [`T`]. Returns `None` otherwise. + pub fn $method_name(&self) -> Option<$type> { + match self { + Primitive::$variant(value) => *value, + _ => None, + } } - } + }; +} - /// If the `Primitive` is a usize, returns the associated [`u32`]. Returns `None` otherwise. - pub fn as_usize(&self) -> Option { - match self { - Primitive::USize(value) => *value, - _ => None, - } - } +impl Primitive { + get_primitive!(as_u8, U8, u8); + get_primitive!(as_u16, U16, u16); + get_primitive!(as_u32, U32, u32); + get_primitive!(as_u64, U64, u64); + get_primitive!(as_u128, U128, u128); + get_primitive!(as_u256, U256, U256); + get_primitive!(as_bool, Bool, bool); + get_primitive!(as_usize, USize, u32); + get_primitive!(as_felt252, Felt252, FieldElement); + get_primitive!(as_class_hash, ClassHash, FieldElement); + get_primitive!(as_contract_address, ContractAddress, FieldElement); - /// If the `Primitive` is a bool, returns the associated [`bool`]. Returns `None` otherwise. - pub fn as_bool(&self) -> Option { - match self { - Primitive::Bool(value) => *value, - _ => None, - } - } + set_primitive!(set_u8, U8, u8); + set_primitive!(set_u16, U16, u16); + set_primitive!(set_u32, U32, u32); + set_primitive!(set_u64, U64, u64); + set_primitive!(set_u128, U128, u128); + set_primitive!(set_u256, U256, U256); + set_primitive!(set_bool, Bool, bool); + set_primitive!(set_usize, USize, u32); + set_primitive!(set_felt252, Felt252, FieldElement); + set_primitive!(set_class_hash, ClassHash, FieldElement); + set_primitive!(set_contract_address, ContractAddress, FieldElement); pub fn to_sql_type(&self) -> SqlType { match self { @@ -333,28 +297,39 @@ mod tests { } #[test] - fn as_inner_value() { - let primitive = Primitive::U8(Some(1u8)); + fn inner_value_getter_setter() { + let mut primitive = Primitive::U8(None); + primitive.set_u8(Some(1u8)).unwrap(); assert_eq!(primitive.as_u8(), Some(1u8)); - let primitive = Primitive::U16(Some(1u16)); + let mut primitive = Primitive::U16(None); + primitive.set_u16(Some(1u16)).unwrap(); assert_eq!(primitive.as_u16(), Some(1u16)); - let primitive = Primitive::U32(Some(1u32)); + let mut primitive = Primitive::U32(None); + primitive.set_u32(Some(1u32)).unwrap(); assert_eq!(primitive.as_u32(), Some(1u32)); - let primitive = Primitive::U64(Some(1u64)); + let mut primitive = Primitive::U64(None); + primitive.set_u64(Some(1u64)).unwrap(); assert_eq!(primitive.as_u64(), Some(1u64)); - let primitive = Primitive::U128(Some(1u128)); + let mut primitive = Primitive::U128(None); + primitive.set_u128(Some(1u128)).unwrap(); assert_eq!(primitive.as_u128(), Some(1u128)); - let primitive = Primitive::U256(Some(U256::from(1u128))); + let mut primitive = Primitive::U256(None); + primitive.set_u256(Some(U256::from(1u128))).unwrap(); assert_eq!(primitive.as_u256(), Some(U256::from(1u128))); - let primitive = Primitive::USize(Some(1u32)); + let mut primitive = Primitive::USize(None); + primitive.set_usize(Some(1u32)).unwrap(); assert_eq!(primitive.as_usize(), Some(1u32)); - let primitive = Primitive::Bool(Some(true)); + let mut primitive = Primitive::Bool(None); + primitive.set_bool(Some(true)).unwrap(); assert_eq!(primitive.as_bool(), Some(true)); - let primitive = Primitive::Felt252(Some(FieldElement::from(1u128))); + let mut primitive = Primitive::Felt252(None); + primitive.set_felt252(Some(FieldElement::from(1u128))).unwrap(); assert_eq!(primitive.as_felt252(), Some(FieldElement::from(1u128))); - let primitive = Primitive::ClassHash(Some(FieldElement::from(1u128))); + let mut primitive = Primitive::ClassHash(None); + primitive.set_class_hash(Some(FieldElement::from(1u128))).unwrap(); assert_eq!(primitive.as_class_hash(), Some(FieldElement::from(1u128))); - let primitive = Primitive::ContractAddress(Some(FieldElement::from(1u128))); + let mut primitive = Primitive::ContractAddress(None); + primitive.set_contract_address(Some(FieldElement::from(1u128))).unwrap(); assert_eq!(primitive.as_contract_address(), Some(FieldElement::from(1u128))); } } diff --git a/crates/dojo-types/src/schema.rs b/crates/dojo-types/src/schema.rs index ec86efd39c..2c0fed749c 100644 --- a/crates/dojo-types/src/schema.rs +++ b/crates/dojo-types/src/schema.rs @@ -231,6 +231,10 @@ impl Struct { pub fn keys(&self) -> Vec { self.children.iter().filter(|m| m.key).cloned().collect() } + + pub fn has_nested(&self) -> bool { + self.children.iter().any(|child| matches!(child.ty, Ty::Struct(_))) + } } #[derive(Debug, thiserror::Error)] @@ -269,6 +273,16 @@ impl Enum { Ok(self.options[option].name.clone()) } + pub fn set_option(&mut self, name: &str) -> Result<(), EnumError> { + match self.options.iter().position(|option| option.name == name) { + Some(index) => { + self.option = Some(index as u8); + Ok(()) + } + None => Err(EnumError::OptionInvalid), + } + } + pub fn to_sql_value(&self) -> Result { self.option() } diff --git a/crates/torii/core/Cargo.toml b/crates/torii/core/Cargo.toml index d6020e930d..9ad2757856 100644 --- a/crates/torii/core/Cargo.toml +++ b/crates/torii/core/Cargo.toml @@ -13,6 +13,7 @@ anyhow.workspace = true async-trait.workspace = true base64.workspace = true chrono.workspace = true +crypto-bigint = { version = "0.5.3", features = [ "serde" ] } dojo-types = { path = "../../dojo-types" } dojo-world = { path = "../../dojo-world", features = [ "contracts", "manifest" ] } futures-channel = "0.3.0" diff --git a/crates/torii/core/src/error.rs b/crates/torii/core/src/error.rs index cdcf6b9b95..1cb4061559 100644 --- a/crates/torii/core/src/error.rs +++ b/crates/torii/core/src/error.rs @@ -1,3 +1,7 @@ +use std::num::ParseIntError; + +use dojo_types::primitive::PrimitiveError; +use dojo_types::schema::EnumError; use starknet::core::types::{FromByteSliceError, FromStrError}; use starknet::core::utils::CairoShortStringToFeltError; @@ -9,6 +13,10 @@ pub enum Error { Sql(#[from] sqlx::Error), #[error(transparent)] QueryError(#[from] QueryError), + #[error(transparent)] + PrimitiveError(#[from] PrimitiveError), + #[error(transparent)] + EnumError(#[from] EnumError), } #[derive(Debug, thiserror::Error)] @@ -19,10 +27,14 @@ pub enum ParseError { CairoShortStringToFelt(#[from] CairoShortStringToFeltError), #[error(transparent)] FromByteSliceError(#[from] FromByteSliceError), + #[error(transparent)] + ParseIntError(#[from] ParseIntError), } #[derive(Debug, thiserror::Error)] pub enum QueryError { #[error("unsupported query")] UnsupportedQuery, + #[error("model not found: {0}")] + ModelNotFound(String), } diff --git a/crates/torii/core/src/model.rs b/crates/torii/core/src/model.rs index e701cf28e0..010fe4310b 100644 --- a/crates/torii/core/src/model.rs +++ b/crates/torii/core/src/model.rs @@ -1,10 +1,18 @@ +use std::str::FromStr; + use async_trait::async_trait; +use crypto_bigint::U256; +use dojo_types::primitive::Primitive; use dojo_types::schema::{Enum, EnumOption, Member, Struct, Ty}; use dojo_world::contracts::model::ModelReader; -use sqlx::{Pool, Sqlite}; +use serde_json::Value; +use sqlx::sqlite::SqliteRow; +use sqlx::{Pool, QueryBuilder, Row, Sqlite}; use starknet::core::types::FieldElement; +use starknet::macros::felt; use super::error::{self, Error}; +use crate::error::{ParseError, QueryError}; pub struct ModelSQLReader { /// The name of the model @@ -144,11 +152,154 @@ pub fn parse_sql_model_members(model: &str, model_members_all: &[SqlModelMember] parse_sql_model_members_impl(model, model_members_all) } +/// A helper function to build a model query including all nested structs +pub fn build_sql_model_query(schema: &Struct) -> String { + fn build_sql_model_query_impl( + path: &str, + schema: &Struct, + selections: &mut Vec, + tables: &mut Vec, + ) { + for child in &schema.children { + match &child.ty { + Ty::Struct(s) => { + let table_name = format!("{}${}", path, s.name); + build_sql_model_query_impl(&table_name, &s, selections, tables); + + tables.push(table_name); + } + _ => { + // alias selected columns to avoid conflicts in JOIN + selections.push(format!( + "{}.external_{} AS \"{}.{}\"", + path, child.name, path, child.name + )); + } + } + } + } + + let model_table = &schema.name; + let mut selections = Vec::new(); + let mut tables = Vec::new(); + + build_sql_model_query_impl(model_table, schema, &mut selections, &mut tables); + let selections_clause = selections.join(", "); + let join_clause = tables + .into_iter() + .map(|table| { + format!("LEFT JOIN {} ON {}.entity_id = {}.entity_id", table, model_table, table) + }) + .collect::>() + .join(" "); + + format!( + "SELECT {selections_clause} FROM {model_table} {join_clause} ORDER BY \ + {model_table}.event_id" + ) +} + +/// Converts SQLite rows into a vector of `Ty` based on a specified schema. +pub fn map_rows_to_tys(schema: &Struct, rows: &Vec) -> Result, Error> { + fn populate_struct_from_row( + path: &str, + struct_ty: &mut Struct, + row: &SqliteRow, + ) -> Result<(), Error> { + for child in struct_ty.children.iter_mut() { + let column_name = format!("{}.{}", path, child.name); + match &mut child.ty { + Ty::Primitive(p) => { + match &p { + Primitive::Bool(_) => { + let value = row.try_get::(&column_name)?; + p.set_bool(Some(value == 1))?; + } + Primitive::USize(_) => { + let value = row.try_get::(&column_name)?; + p.set_usize(Some(value as u32))?; + } + Primitive::U8(_) => { + let value = row.try_get::(&column_name)?; + p.set_u8(Some(value as u8))?; + } + Primitive::U16(_) => { + let value = row.try_get::(&column_name)?; + p.set_u16(Some(value as u16))?; + } + Primitive::U32(_) => { + let value = row.try_get::(&column_name)?; + p.set_u32(Some(value as u32))?; + } + Primitive::U64(_) => { + let value = row.try_get::(&column_name)?; + p.set_u64(Some(value as u64))?; + } + Primitive::U128(_) => { + let value = row.try_get::(&column_name)?; + let hex_str = value.trim_start_matches("0x"); + p.set_u128(Some( + u128::from_str_radix(hex_str, 16) + .map_err(ParseError::ParseIntError)?, + ))?; + } + Primitive::U256(_) => { + let value = row.try_get::(&column_name)?; + let hex_str = value.trim_start_matches("0x"); + p.set_u256(Some(U256::from_be_hex(hex_str)))?; + } + Primitive::Felt252(_) => { + let value = row.try_get::(&column_name)?; + p.set_felt252(Some( + FieldElement::from_str(&value).map_err(ParseError::FromStr)?, + ))?; + } + Primitive::ClassHash(_) => { + let value = row.try_get::(&column_name)?; + p.set_class_hash(Some( + FieldElement::from_str(&value).map_err(ParseError::FromStr)?, + ))?; + } + Primitive::ContractAddress(_) => { + let value = row.try_get::(&column_name)?; + p.set_contract_address(Some( + FieldElement::from_str(&value).map_err(ParseError::FromStr)?, + ))?; + } + }; + } + Ty::Enum(e) => { + let value = row.try_get::(&column_name)?; + e.set_option(&value)?; + } + Ty::Struct(nested) => { + let path = [path, &nested.name].join("$"); + populate_struct_from_row(&path, nested, row)?; + } + ty => { + unimplemented!("unimplemented type_enum: {ty}"); + } + }; + } + + Ok(()) + } + + rows.iter() + .map(|row| { + let mut struct_ty = schema.clone(); + populate_struct_from_row(&schema.name, &mut struct_ty, row)?; + + Ok(Ty::Struct(struct_ty)) + }) + .collect::, Error>>() +} + #[cfg(test)] mod tests { use dojo_types::schema::{Enum, EnumOption, Member, Struct, Ty}; - use super::SqlModelMember; + use super::{build_sql_model_query, SqlModelMember}; use crate::model::parse_sql_model_members; #[test] @@ -321,4 +472,52 @@ mod tests { assert_eq!(parse_sql_model_members("Moves", &model_members), expected_ty); } + + #[test] + fn struct_ty_to_query() { + let ty = Ty::Struct(Struct { + name: "Position".into(), + children: vec![ + dojo_types::schema::Member { + name: "name".into(), + key: false, + ty: Ty::Primitive("felt252".parse().unwrap()), + }, + dojo_types::schema::Member { + name: "age".into(), + key: false, + ty: Ty::Primitive("u8".parse().unwrap()), + }, + dojo_types::schema::Member { + name: "vec".into(), + key: false, + ty: Ty::Struct(Struct { + name: "Vec2".into(), + children: vec![ + Member { + name: "x".into(), + key: false, + ty: Ty::Primitive("u256".parse().unwrap()), + }, + Member { + name: "y".into(), + key: false, + ty: Ty::Primitive("u256".parse().unwrap()), + }, + ], + }), + }, + ], + }); + + let query = build_sql_model_query(ty.as_struct().unwrap()); + assert_eq!( + query, + "SELECT Position.external_name AS \"Position.name\", Position.external_age AS \ + \"Position.age\", Position$Vec2.external_x AS \"Position$Vec2.x\", \ + Position$Vec2.external_y AS \"Position$Vec2.y\" FROM Position LEFT JOIN \ + Position$Vec2 ON Position.entity_id = Position$Vec2.entity_id ORDER BY \ + Position.event_id" + ); + } } diff --git a/crates/torii/grpc/proto/types.proto b/crates/torii/grpc/proto/types.proto index 1c853c005b..d9dab2f1f3 100644 --- a/crates/torii/grpc/proto/types.proto +++ b/crates/torii/grpc/proto/types.proto @@ -29,6 +29,30 @@ message ModelMetadata { bytes schema = 6; } +message Member { + // Name of the member + string name = 1; + // Type of value + oneof value_type { + Value value = 2; + Model struct = 3; + } +} + +message Model { + // Name of the model + string name = 1; + // Members of the model + repeated Member members = 2; +} + +message Entity { + // The entity key + string key = 1; + // Models of the entity + repeated Model models = 2; +} + message StorageEntry { // The key of the changed value string key = 1; @@ -72,7 +96,7 @@ message KeysClause { message AttributeClause { string model = 1; - string attribute = 2; + string member = 2; ComparisonOperator operator = 3; Value value = 4; } diff --git a/crates/torii/grpc/proto/world.proto b/crates/torii/grpc/proto/world.proto index 66f5fce09d..ec6dccc469 100644 --- a/crates/torii/grpc/proto/world.proto +++ b/crates/torii/grpc/proto/world.proto @@ -9,8 +9,11 @@ service World { rpc WorldMetadata (MetadataRequest) returns (MetadataResponse); - // Subscribes to entity updates. + // Subscribes to entities updates. rpc SubscribeEntities (SubscribeEntitiesRequest) returns (stream SubscribeEntitiesResponse); + + // Retrieve entity + rpc RetrieveEntities (RetrieveEntitiesRequest) returns (RetrieveEntitiesResponse); } @@ -33,3 +36,12 @@ message SubscribeEntitiesResponse { // List of entities that have been updated. types.EntityUpdate entity_update = 1; } + +message RetrieveEntitiesRequest { + // The entities to retrieve + types.EntityQuery query = 1; +} + +message RetrieveEntitiesResponse { + repeated types.Entity entities = 1; +} diff --git a/crates/torii/grpc/src/server/mod.rs b/crates/torii/grpc/src/server/mod.rs index d5ccac1680..ac9971b861 100644 --- a/crates/torii/grpc/src/server/mod.rs +++ b/crates/torii/grpc/src/server/mod.rs @@ -7,11 +7,13 @@ use std::net::SocketAddr; use std::pin::Pin; use std::sync::Arc; +use dojo_types::schema::Ty; use futures::Stream; use proto::world::{ - MetadataRequest, MetadataResponse, SubscribeEntitiesRequest, SubscribeEntitiesResponse, + MetadataRequest, MetadataResponse, RetrieveEntitiesRequest, RetrieveEntitiesResponse, + SubscribeEntitiesRequest, SubscribeEntitiesResponse, }; -use sqlx::{Pool, Sqlite}; +use sqlx::{Pool, Row, Sqlite}; use starknet::core::utils::cairo_short_string_to_felt; use starknet::providers::jsonrpc::HttpTransport; use starknet::providers::JsonRpcClient; @@ -21,10 +23,13 @@ use tokio::sync::mpsc::Receiver; use tokio_stream::wrappers::{ReceiverStream, TcpListenerStream}; use tonic::transport::Server; use tonic::{Request, Response, Status}; -use torii_core::error::{Error, ParseError}; -use torii_core::model::{parse_sql_model_members, SqlModelMember}; +use torii_core::error::{Error, ParseError, QueryError}; +use torii_core::model::{ + build_sql_model_query, map_rows_to_tys, parse_sql_model_members, SqlModelMember, +}; use self::subscription::SubscribeRequest; +use crate::proto::types::clause::ClauseType; use crate::proto::world::world_server::WorldServer; use crate::proto::{self}; @@ -107,9 +112,44 @@ impl DojoWorld { .fetch_all(&self.pool) .await?; + if model_members.is_empty() { + return Err(QueryError::ModelNotFound(model.into()).into()); + } + Ok(parse_sql_model_members(model, &model_members)) } + async fn entities_by_keys( + &self, + keys: proto::types::KeysClause, + ) -> Result, Error> { + Ok(vec![]) + } + + async fn entities_by_attribute( + &self, + attribute: proto::types::AttributeClause, + ) -> Result, Error> { + // TODO: cache model schema and query to avoid rebuilding everytime + let ty = self.model_schema(&attribute.model).await?; + let schema = ty.as_struct().ok_or(QueryError::UnsupportedQuery)?; + let query = build_sql_model_query(schema); + + let results = sqlx::query(&query).fetch_all(&self.pool).await?; + let tys = map_rows_to_tys(schema, &results)?; + + println!("{:#?}", tys); + + Ok(vec![]) + } + + async fn entities_by_composite( + &self, + composite: proto::types::CompositeClause, + ) -> Result, Error> { + Ok(vec![]) + } + pub async fn model_metadata(&self, model: &str) -> Result { let (name, class_hash, packed_size, unpacked_size, layout): ( String, @@ -161,6 +201,25 @@ impl DojoWorld { self.subscriber_manager.add_subscriber(subs).await } + + async fn retrieve_entities( + &self, + query: proto::types::EntityQuery, + ) -> Result { + let clause_type = query + .clause + .ok_or(QueryError::UnsupportedQuery)? + .clause_type + .ok_or(QueryError::UnsupportedQuery)?; + + let entities = match clause_type { + ClauseType::Keys(keys) => self.entities_by_keys(keys).await?, + ClauseType::Attribute(attribute) => self.entities_by_attribute(attribute).await?, + ClauseType::Composite(composite) => self.entities_by_composite(composite).await?, + }; + + Ok(RetrieveEntitiesResponse { entities }) + } } type ServiceResult = Result, Status>; @@ -194,6 +253,21 @@ impl proto::world::world_server::World for DojoWorld { .map_err(|e| Status::internal(e.to_string()))?; Ok(Response::new(Box::pin(ReceiverStream::new(rx)) as Self::SubscribeEntitiesStream)) } + + async fn retrieve_entities( + &self, + request: Request, + ) -> Result, Status> { + let query = request + .into_inner() + .query + .ok_or_else(|| Status::invalid_argument("Missing query argument"))?; + + let entities = + self.retrieve_entities(query).await.map_err(|e| Status::internal(e.to_string()))?; + + Ok(Response::new(entities)) + } } pub async fn new( diff --git a/crates/torii/grpc/src/types.rs b/crates/torii/grpc/src/types.rs index e62dc3b8a8..ef147bad36 100644 --- a/crates/torii/grpc/src/types.rs +++ b/crates/torii/grpc/src/types.rs @@ -31,7 +31,7 @@ pub struct KeysClause { #[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] pub struct AttributeClause { pub model: String, - pub attribute: String, + pub member: String, pub operator: ComparisonOperator, pub value: Value, } @@ -152,7 +152,7 @@ impl From for proto::types::AttributeClause { fn from(value: AttributeClause) -> Self { Self { model: value.model, - attribute: value.attribute, + member: value.member, operator: value.operator as i32, value: Some(value.value.into()), } diff --git a/scripts/rust_fmt.sh b/scripts/rust_fmt.sh index 62a418693a..095c105201 100755 --- a/scripts/rust_fmt.sh +++ b/scripts/rust_fmt.sh @@ -1,3 +1,3 @@ #!/bin/bash -cargo +nightly fmt --check --all -- "$@" +cargo +nightly fmt --all -- "$@" From 93d194a107cbd267f35bd55dca34d9f8814f7fc6 Mon Sep 17 00:00:00 2001 From: broody Date: Sun, 19 Nov 2023 10:20:42 -0800 Subject: [PATCH 09/16] Add cache to store schema ty and query --- crates/torii/core/src/cache.rs | 63 +++++++++++++++++++++++++++++ crates/torii/core/src/lib.rs | 1 + crates/torii/core/src/model.rs | 14 +++---- crates/torii/grpc/src/server/mod.rs | 53 ++++++++---------------- 4 files changed, 86 insertions(+), 45 deletions(-) create mode 100644 crates/torii/core/src/cache.rs diff --git a/crates/torii/core/src/cache.rs b/crates/torii/core/src/cache.rs new file mode 100644 index 0000000000..812ce20972 --- /dev/null +++ b/crates/torii/core/src/cache.rs @@ -0,0 +1,63 @@ +use std::collections::HashMap; + +use dojo_types::schema::Ty; +use sqlx::SqlitePool; +use tokio::sync::RwLock; + +use crate::error::{Error, QueryError}; +use crate::model::{build_sql_model_query, parse_sql_model_members, SqlModelMember}; + +#[derive(Clone)] +pub struct SchemaInfo { + pub ty: Ty, + pub query: String, +} + +pub struct ModelCache { + pool: SqlitePool, + schemas: RwLock>, +} + +impl ModelCache { + pub fn new(pool: SqlitePool) -> Self { + Self { pool, schemas: RwLock::new(HashMap::new()) } + } + + pub async fn schema(&self, model: &str) -> Result { + { + let schemas = self.schemas.read().await; + if let Some(schema_info) = schemas.get(model) { + return Ok(schema_info.clone()); + } + } + + self.update(model).await + } + + async fn update(&self, model: &str) -> Result { + let model_members: Vec = sqlx::query_as( + "SELECT id, model_idx, member_idx, name, type, type_enum, enum_options, key FROM \ + model_members WHERE model_id = ? ORDER BY model_idx ASC, member_idx ASC", + ) + .bind(model) + .fetch_all(&self.pool) + .await?; + + if model_members.is_empty() { + return Err(QueryError::ModelNotFound(model.into()).into()); + } + + let ty = parse_sql_model_members(model, &model_members); + let query = build_sql_model_query(ty.as_struct().unwrap()); + let schema_info = SchemaInfo { ty, query }; + + let mut schemas = self.schemas.write().await; + schemas.insert(model.into(), schema_info.clone()); + + Ok(schema_info) + } + + pub async fn clear(&self) { + self.schemas.write().await.clear(); + } +} diff --git a/crates/torii/core/src/lib.rs b/crates/torii/core/src/lib.rs index 877aab65a9..e36c3f2e3b 100644 --- a/crates/torii/core/src/lib.rs +++ b/crates/torii/core/src/lib.rs @@ -3,6 +3,7 @@ use sqlx::FromRow; use crate::types::SQLFieldElement; +pub mod cache; pub mod engine; pub mod error; pub mod model; diff --git a/crates/torii/core/src/model.rs b/crates/torii/core/src/model.rs index 010fe4310b..41cf9b4af7 100644 --- a/crates/torii/core/src/model.rs +++ b/crates/torii/core/src/model.rs @@ -5,14 +5,12 @@ use crypto_bigint::U256; use dojo_types::primitive::Primitive; use dojo_types::schema::{Enum, EnumOption, Member, Struct, Ty}; use dojo_world::contracts::model::ModelReader; -use serde_json::Value; use sqlx::sqlite::SqliteRow; -use sqlx::{Pool, QueryBuilder, Row, Sqlite}; +use sqlx::{Pool, Row, Sqlite}; use starknet::core::types::FieldElement; -use starknet::macros::felt; use super::error::{self, Error}; -use crate::error::{ParseError, QueryError}; +use crate::error::ParseError; pub struct ModelSQLReader { /// The name of the model @@ -164,7 +162,7 @@ pub fn build_sql_model_query(schema: &Struct) -> String { match &child.ty { Ty::Struct(s) => { let table_name = format!("{}${}", path, s.name); - build_sql_model_query_impl(&table_name, &s, selections, tables); + build_sql_model_query_impl(&table_name, s, selections, tables); tables.push(table_name); } @@ -188,19 +186,19 @@ pub fn build_sql_model_query(schema: &Struct) -> String { let join_clause = tables .into_iter() .map(|table| { - format!("LEFT JOIN {} ON {}.entity_id = {}.entity_id", table, model_table, table) + format!(" LEFT JOIN {} ON {}.entity_id = {}.entity_id", table, model_table, table) }) .collect::>() .join(" "); format!( - "SELECT {selections_clause} FROM {model_table} {join_clause} ORDER BY \ + "SELECT {selections_clause} FROM {model_table}{join_clause} ORDER BY \ {model_table}.event_id" ) } /// Converts SQLite rows into a vector of `Ty` based on a specified schema. -pub fn map_rows_to_tys(schema: &Struct, rows: &Vec) -> Result, Error> { +pub fn map_rows_to_tys(schema: &Struct, rows: &[SqliteRow]) -> Result, Error> { fn populate_struct_from_row( path: &str, struct_ty: &mut Struct, diff --git a/crates/torii/grpc/src/server/mod.rs b/crates/torii/grpc/src/server/mod.rs index ac9971b861..f5edcc3af3 100644 --- a/crates/torii/grpc/src/server/mod.rs +++ b/crates/torii/grpc/src/server/mod.rs @@ -7,13 +7,12 @@ use std::net::SocketAddr; use std::pin::Pin; use std::sync::Arc; -use dojo_types::schema::Ty; use futures::Stream; use proto::world::{ MetadataRequest, MetadataResponse, RetrieveEntitiesRequest, RetrieveEntitiesResponse, SubscribeEntitiesRequest, SubscribeEntitiesResponse, }; -use sqlx::{Pool, Row, Sqlite}; +use sqlx::{Pool, Sqlite}; use starknet::core::utils::cairo_short_string_to_felt; use starknet::providers::jsonrpc::HttpTransport; use starknet::providers::JsonRpcClient; @@ -23,10 +22,9 @@ use tokio::sync::mpsc::Receiver; use tokio_stream::wrappers::{ReceiverStream, TcpListenerStream}; use tonic::transport::Server; use tonic::{Request, Response, Status}; +use torii_core::cache::ModelCache; use torii_core::error::{Error, ParseError, QueryError}; -use torii_core::model::{ - build_sql_model_query, map_rows_to_tys, parse_sql_model_members, SqlModelMember, -}; +use torii_core::model::map_rows_to_tys; use self::subscription::SubscribeRequest; use crate::proto::types::clause::ClauseType; @@ -38,6 +36,7 @@ pub struct DojoWorld { world_address: FieldElement, pool: Pool, subscriber_manager: Arc, + model_cache: Arc, } impl DojoWorld { @@ -56,7 +55,9 @@ impl DojoWorld { Arc::clone(&subscriber_manager), )); - Self { pool, world_address, subscriber_manager } + let model_cache = Arc::new(ModelCache::new(pool.clone())); + + Self { pool, model_cache, world_address, subscriber_manager } } } @@ -83,14 +84,14 @@ impl DojoWorld { let mut models_metadata = Vec::with_capacity(models.len()); for model in models { - let schema = self.model_schema(&model.0).await?; + let schema_info = self.model_cache.schema(&model.0).await?; models_metadata.push(proto::types::ModelMetadata { name: model.0, class_hash: model.1, packed_size: model.2, unpacked_size: model.3, layout: hex::decode(&model.4).unwrap(), - schema: serde_json::to_vec(&schema).unwrap(), + schema: serde_json::to_vec(&schema_info.ty).unwrap(), }); } @@ -103,25 +104,9 @@ impl DojoWorld { }) } - async fn model_schema(&self, model: &str) -> Result { - let model_members: Vec = sqlx::query_as( - "SELECT id, model_idx, member_idx, name, type, type_enum, enum_options, key FROM \ - model_members WHERE model_id = ? ORDER BY model_idx ASC, member_idx ASC", - ) - .bind(model) - .fetch_all(&self.pool) - .await?; - - if model_members.is_empty() { - return Err(QueryError::ModelNotFound(model.into()).into()); - } - - Ok(parse_sql_model_members(model, &model_members)) - } - async fn entities_by_keys( &self, - keys: proto::types::KeysClause, + _keys: proto::types::KeysClause, ) -> Result, Error> { Ok(vec![]) } @@ -130,22 +115,16 @@ impl DojoWorld { &self, attribute: proto::types::AttributeClause, ) -> Result, Error> { - // TODO: cache model schema and query to avoid rebuilding everytime - let ty = self.model_schema(&attribute.model).await?; - let schema = ty.as_struct().ok_or(QueryError::UnsupportedQuery)?; - let query = build_sql_model_query(schema); - - let results = sqlx::query(&query).fetch_all(&self.pool).await?; - let tys = map_rows_to_tys(schema, &results)?; - - println!("{:#?}", tys); + let schema_info = self.model_cache.schema(&attribute.model).await?; + let results = sqlx::query(&schema_info.query).fetch_all(&self.pool).await?; + let _ = map_rows_to_tys(schema_info.ty.as_struct().unwrap(), &results)?; Ok(vec![]) } async fn entities_by_composite( &self, - composite: proto::types::CompositeClause, + _composite: proto::types::CompositeClause, ) -> Result, Error> { Ok(vec![]) } @@ -164,7 +143,7 @@ impl DojoWorld { .fetch_one(&self.pool) .await?; - let schema = self.model_schema(model).await?; + let schema = self.model_cache.schema(model).await?; let layout = hex::decode(&layout).unwrap(); Ok(proto::types::ModelMetadata { @@ -173,7 +152,7 @@ impl DojoWorld { class_hash, packed_size, unpacked_size, - schema: serde_json::to_vec(&schema).unwrap(), + schema: serde_json::to_vec(&schema.ty).unwrap(), }) } From 7f4d9028eeaf3fcda0245106cb5b5c02bc1bb373 Mon Sep 17 00:00:00 2001 From: broody Date: Mon, 20 Nov 2023 14:03:26 -0800 Subject: [PATCH 10/16] map rows to proto model --- crates/dojo-types/src/primitive.rs | 24 +++--- crates/torii/core/src/cache.rs | 10 +-- crates/torii/core/src/model.rs | 60 ++++++------- crates/torii/grpc/proto/types.proto | 12 ++- crates/torii/grpc/src/server/mod.rs | 129 +++++++++++++++++++++++----- scripts/rust_fmt.sh | 2 +- 6 files changed, 164 insertions(+), 73 deletions(-) diff --git a/crates/dojo-types/src/primitive.rs b/crates/dojo-types/src/primitive.rs index dbb266df86..560dfc151c 100644 --- a/crates/dojo-types/src/primitive.rs +++ b/crates/dojo-types/src/primitive.rs @@ -63,7 +63,7 @@ macro_rules! set_primitive { } /// Macro to generate getter methods for Primitive enum variants. -macro_rules! get_primitive { +macro_rules! as_primitive { ($method_name:ident, $variant:ident, $type:ty) => { /// If the `Primitive` is type T, returns the associated [`T`]. Returns `None` otherwise. pub fn $method_name(&self) -> Option<$type> { @@ -76,17 +76,17 @@ macro_rules! get_primitive { } impl Primitive { - get_primitive!(as_u8, U8, u8); - get_primitive!(as_u16, U16, u16); - get_primitive!(as_u32, U32, u32); - get_primitive!(as_u64, U64, u64); - get_primitive!(as_u128, U128, u128); - get_primitive!(as_u256, U256, U256); - get_primitive!(as_bool, Bool, bool); - get_primitive!(as_usize, USize, u32); - get_primitive!(as_felt252, Felt252, FieldElement); - get_primitive!(as_class_hash, ClassHash, FieldElement); - get_primitive!(as_contract_address, ContractAddress, FieldElement); + as_primitive!(as_u8, U8, u8); + as_primitive!(as_u16, U16, u16); + as_primitive!(as_u32, U32, u32); + as_primitive!(as_u64, U64, u64); + as_primitive!(as_u128, U128, u128); + as_primitive!(as_u256, U256, U256); + as_primitive!(as_bool, Bool, bool); + as_primitive!(as_usize, USize, u32); + as_primitive!(as_felt252, Felt252, FieldElement); + as_primitive!(as_class_hash, ClassHash, FieldElement); + as_primitive!(as_contract_address, ContractAddress, FieldElement); set_primitive!(set_u8, U8, u8); set_primitive!(set_u16, U16, u16); diff --git a/crates/torii/core/src/cache.rs b/crates/torii/core/src/cache.rs index a67ba5befc..0296b46d25 100644 --- a/crates/torii/core/src/cache.rs +++ b/crates/torii/core/src/cache.rs @@ -26,8 +26,8 @@ impl ModelCache { pub async fn schema(&self, model: &str) -> Result, Error> { { let schemas = self.schemas.read().await; - if let Some(schema_data) = schemas.get(model) { - return Ok(Arc::clone(schema_data)); + if let Some(schema) = schemas.get(model) { + return Ok(Arc::clone(schema)); } } @@ -49,12 +49,12 @@ impl ModelCache { let ty = parse_sql_model_members(model, &model_members); let sql = build_sql_model_query(ty.as_struct().unwrap()); - let schema_data = Arc::new(SchemaData { ty, sql }); + let schema = Arc::new(SchemaData { ty, sql }); let mut schemas = self.schemas.write().await; - schemas.insert(model.into(), Arc::clone(&schema_data)); + schemas.insert(model.into(), Arc::clone(&schema)); - Ok(schema_data) + Ok(schema) } pub async fn clear(&self) { diff --git a/crates/torii/core/src/model.rs b/crates/torii/core/src/model.rs index 41cf9b4af7..433cd29cb8 100644 --- a/crates/torii/core/src/model.rs +++ b/crates/torii/core/src/model.rs @@ -150,7 +150,7 @@ pub fn parse_sql_model_members(model: &str, model_members_all: &[SqlModelMember] parse_sql_model_members_impl(model, model_members_all) } -/// A helper function to build a model query including all nested structs +/// A helper function to build a model query including all nested structs and its the entity id pub fn build_sql_model_query(schema: &Struct) -> String { fn build_sql_model_query_impl( path: &str, @@ -198,45 +198,45 @@ pub fn build_sql_model_query(schema: &Struct) -> String { } /// Converts SQLite rows into a vector of `Ty` based on a specified schema. -pub fn map_rows_to_tys(schema: &Struct, rows: &[SqliteRow]) -> Result, Error> { +pub fn map_rows_to_tys(schema: &Ty, rows: &[SqliteRow]) -> Result, Error> { fn populate_struct_from_row( path: &str, struct_ty: &mut Struct, row: &SqliteRow, ) -> Result<(), Error> { - for child in struct_ty.children.iter_mut() { - let column_name = format!("{}.{}", path, child.name); - match &mut child.ty { - Ty::Primitive(p) => { - match &p { + for member in struct_ty.children.iter_mut() { + let column_name = format!("{}.{}", path, member.name); + match &mut member.ty { + Ty::Primitive(primitive) => { + match &primitive { Primitive::Bool(_) => { - let value = row.try_get::(&column_name)?; - p.set_bool(Some(value == 1))?; + let value = row.try_get::(&column_name)?; + primitive.set_bool(Some(value))?; } Primitive::USize(_) => { - let value = row.try_get::(&column_name)?; - p.set_usize(Some(value as u32))?; + let value = row.try_get::(&column_name)?; + primitive.set_usize(Some(value))?; } Primitive::U8(_) => { - let value = row.try_get::(&column_name)?; - p.set_u8(Some(value as u8))?; + let value = row.try_get::(&column_name)?; + primitive.set_u8(Some(value))?; } Primitive::U16(_) => { - let value = row.try_get::(&column_name)?; - p.set_u16(Some(value as u16))?; + let value = row.try_get::(&column_name)?; + primitive.set_u16(Some(value))?; } Primitive::U32(_) => { - let value = row.try_get::(&column_name)?; - p.set_u32(Some(value as u32))?; + let value = row.try_get::(&column_name)?; + primitive.set_u32(Some(value))?; } Primitive::U64(_) => { let value = row.try_get::(&column_name)?; - p.set_u64(Some(value as u64))?; + primitive.set_u64(Some(value as u64))?; } Primitive::U128(_) => { let value = row.try_get::(&column_name)?; let hex_str = value.trim_start_matches("0x"); - p.set_u128(Some( + primitive.set_u128(Some( u128::from_str_radix(hex_str, 16) .map_err(ParseError::ParseIntError)?, ))?; @@ -244,35 +244,35 @@ pub fn map_rows_to_tys(schema: &Struct, rows: &[SqliteRow]) -> Result, E Primitive::U256(_) => { let value = row.try_get::(&column_name)?; let hex_str = value.trim_start_matches("0x"); - p.set_u256(Some(U256::from_be_hex(hex_str)))?; + primitive.set_u256(Some(U256::from_be_hex(hex_str)))?; } Primitive::Felt252(_) => { let value = row.try_get::(&column_name)?; - p.set_felt252(Some( + primitive.set_felt252(Some( FieldElement::from_str(&value).map_err(ParseError::FromStr)?, ))?; } Primitive::ClassHash(_) => { let value = row.try_get::(&column_name)?; - p.set_class_hash(Some( + primitive.set_class_hash(Some( FieldElement::from_str(&value).map_err(ParseError::FromStr)?, ))?; } Primitive::ContractAddress(_) => { let value = row.try_get::(&column_name)?; - p.set_contract_address(Some( + primitive.set_contract_address(Some( FieldElement::from_str(&value).map_err(ParseError::FromStr)?, ))?; } }; } - Ty::Enum(e) => { + Ty::Enum(enum_ty) => { let value = row.try_get::(&column_name)?; - e.set_option(&value)?; + enum_ty.set_option(&value)?; } - Ty::Struct(nested) => { - let path = [path, &nested.name].join("$"); - populate_struct_from_row(&path, nested, row)?; + Ty::Struct(struct_ty) => { + let path = [path, &struct_ty.name].join("$"); + populate_struct_from_row(&path, struct_ty, row)?; } ty => { unimplemented!("unimplemented type_enum: {ty}"); @@ -285,8 +285,8 @@ pub fn map_rows_to_tys(schema: &Struct, rows: &[SqliteRow]) -> Result, E rows.iter() .map(|row| { - let mut struct_ty = schema.clone(); - populate_struct_from_row(&schema.name, &mut struct_ty, row)?; + let mut struct_ty = schema.as_struct().expect("schema should be struct ty").clone(); + populate_struct_from_row(&schema.name(), &mut struct_ty, row)?; Ok(Ty::Struct(struct_ty)) }) diff --git a/crates/torii/grpc/proto/types.proto b/crates/torii/grpc/proto/types.proto index d9dab2f1f3..1dbb7b86e9 100644 --- a/crates/torii/grpc/proto/types.proto +++ b/crates/torii/grpc/proto/types.proto @@ -29,13 +29,21 @@ message ModelMetadata { bytes schema = 6; } +message Enum { + // Variant + uint32 option = 1; + // Variants of the enum + repeated string options = 2; +} + message Member { // Name of the member string name = 1; // Type of value - oneof value_type { + oneof member_type { Value value = 2; - Model struct = 3; + Enum enum = 3; + Model struct = 4; } } diff --git a/crates/torii/grpc/src/server/mod.rs b/crates/torii/grpc/src/server/mod.rs index 3d0674f546..05279c820c 100644 --- a/crates/torii/grpc/src/server/mod.rs +++ b/crates/torii/grpc/src/server/mod.rs @@ -7,12 +7,15 @@ use std::net::SocketAddr; use std::pin::Pin; use std::sync::Arc; +use dojo_types::primitive::Primitive; +use dojo_types::schema::{Struct, Ty}; use futures::Stream; use proto::world::{ MetadataRequest, MetadataResponse, RetrieveEntitiesRequest, RetrieveEntitiesResponse, SubscribeEntitiesRequest, SubscribeEntitiesResponse, }; -use sqlx::{Pool, Sqlite}; +use sqlx::sqlite::SqliteRow; +use sqlx::{Pool, Row, Sqlite}; use starknet::core::utils::cairo_short_string_to_felt; use starknet::providers::jsonrpc::HttpTransport; use starknet::providers::JsonRpcClient; @@ -24,7 +27,6 @@ use tonic::transport::Server; use tonic::{Request, Response, Status}; use torii_core::cache::ModelCache; use torii_core::error::{Error, ParseError, QueryError}; -use torii_core::model::map_rows_to_tys; use self::subscription::SubscribeRequest; use crate::proto::types::clause::ClauseType; @@ -84,14 +86,14 @@ impl DojoWorld { let mut models_metadata = Vec::with_capacity(models.len()); for model in models { - let schema_data = self.model_cache.schema(&model.0).await?; + let schema = self.model_cache.schema(&model.0).await?; models_metadata.push(proto::types::ModelMetadata { name: model.0, class_hash: model.1, packed_size: model.2, unpacked_size: model.3, layout: hex::decode(&model.4).unwrap(), - schema: serde_json::to_vec(&schema_data.ty).unwrap(), + schema: serde_json::to_vec(&schema.ty).unwrap(), }); } @@ -115,25 +117,11 @@ impl DojoWorld { &self, attribute: proto::types::AttributeClause, ) -> Result, Error> { - let schema_data = self.model_cache.schema(&attribute.model).await?; - let results = sqlx::query(&schema_data.sql).fetch_all(&self.pool).await?; - let tys = map_rows_to_tys(schema_data.ty.as_struct().unwrap(), &results)?; - - let mut entities = Vec::with_capacity(tys.len()); - - for ty in tys { - entities.push(proto::types::Entity { - key: "".to_string(), - models: vec![ - proto::types::Model { - name: ty.name(), - data: serde_json::to_vec(&ty).unwrap() - } - ] - }) - } - - Ok(vec![]) + let schema = self.model_cache.schema(&attribute.model).await?; + let rows = sqlx::query(&schema.sql).fetch_all(&self.pool).await?; + let models = self.map_rows_to_models(&schema.ty, &rows).await?; + + Ok(vec![proto::types::Entity { key: "".to_string(), models }]) } async fn entities_by_composite( @@ -213,6 +201,101 @@ impl DojoWorld { Ok(RetrieveEntitiesResponse { entities }) } + + async fn map_rows_to_models( + &self, + schema: &Ty, + rows: &[SqliteRow], + ) -> Result, Error> { + fn row_to_model( + path: &str, + struct_ty: &Struct, + row: &SqliteRow, + ) -> Result { + let members = struct_ty + .children + .iter() + .map(|member| { + let column_name = format!("{}.{}", path, member.name); + let name = member.name.clone(); + let member = match &member.ty { + Ty::Primitive(primitive) => { + let value_type = match primitive { + Primitive::Bool(_) => proto::types::value::ValueType::BoolValue( + row.try_get::(&column_name)?, + ), + Primitive::U8(_) + | Primitive::U16(_) + | Primitive::U32(_) + | Primitive::U64(_) + | Primitive::USize(_) => { + let value = row.try_get::(&column_name)?; + proto::types::value::ValueType::UintValue(value as u64) + } + Primitive::U128(_) + | Primitive::U256(_) + | Primitive::Felt252(_) + | Primitive::ClassHash(_) + | Primitive::ContractAddress(_) => { + let value = row.try_get::(&column_name)?; + proto::types::value::ValueType::StringValue(value) + } + }; + + proto::types::Member { + name, + member_type: Some(proto::types::member::MemberType::Value( + proto::types::Value { value_type: Some(value_type) }, + )), + } + } + + Ty::Enum(enum_ty) => { + let value = row.try_get::(&column_name)?; + let options = enum_ty + .options + .iter() + .map(|e| e.name.to_string()) + .collect::>(); + let option = + options.iter().position(|o| o == &value).expect("wrong enum value") + as u32; + proto::types::Member { + name: enum_ty.name.clone(), + member_type: Some(proto::types::member::MemberType::Enum( + proto::types::Enum { option, options }, + )), + } + } + Ty::Struct(struct_ty) => { + let path = [path, &struct_ty.name].join("$"); + proto::types::Member { + name, + member_type: Some(proto::types::member::MemberType::Struct( + row_to_model(&path, struct_ty, row)?, + )), + } + } + ty => { + unimplemented!("unimplemented type_enum: {ty}"); + } + }; + + Ok(member) + }) + .collect::, Error>>()?; + + Ok(proto::types::Model { name: struct_ty.name.clone(), members }) + } + + rows.iter() + .map(|row| { + let struct_ty = schema.as_struct().expect("schema should be struct ty").clone(); + + row_to_model(&schema.name(), &struct_ty, row) + }) + .collect::, Error>>() + } } type ServiceResult = Result, Status>; diff --git a/scripts/rust_fmt.sh b/scripts/rust_fmt.sh index 095c105201..62a418693a 100755 --- a/scripts/rust_fmt.sh +++ b/scripts/rust_fmt.sh @@ -1,3 +1,3 @@ #!/bin/bash -cargo +nightly fmt --all -- "$@" +cargo +nightly fmt --check --all -- "$@" From 2d77544de02b95e834e34b30d2c6d53ee9c9f183 Mon Sep 17 00:00:00 2001 From: broody Date: Mon, 20 Nov 2023 15:14:26 -0800 Subject: [PATCH 11/16] add grpc limit offset --- crates/torii/grpc/proto/types.proto | 8 +++++++- crates/torii/grpc/src/server/mod.rs | 20 ++++++++++++++++++-- crates/torii/grpc/src/types.rs | 15 ++++++++++++++- 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/crates/torii/grpc/proto/types.proto b/crates/torii/grpc/proto/types.proto index 1dbb7b86e9..101acdfd0d 100644 --- a/crates/torii/grpc/proto/types.proto +++ b/crates/torii/grpc/proto/types.proto @@ -39,7 +39,7 @@ message Enum { message Member { // Name of the member string name = 1; - // Type of value + // Type of member oneof member_type { Value value = 2; Enum enum = 3; @@ -87,6 +87,7 @@ message EntityUpdate { message EntityQuery { Clause clause = 1; + LimitOffset limit_offset = 4; } message Clause { @@ -115,6 +116,11 @@ message CompositeClause { repeated Clause clauses = 3; } +message LimitOffset { + uint32 limit = 1; + uint32 offset = 2; +} + enum LogicalOperator { AND = 0; OR = 1; diff --git a/crates/torii/grpc/src/server/mod.rs b/crates/torii/grpc/src/server/mod.rs index 05279c820c..8792ec525f 100644 --- a/crates/torii/grpc/src/server/mod.rs +++ b/crates/torii/grpc/src/server/mod.rs @@ -33,6 +33,9 @@ use crate::proto::types::clause::ClauseType; use crate::proto::world::world_server::WorldServer; use crate::proto::{self}; +const DEFAULT_LIMIT: u32 = 10; +const DEFAULT_OFFSET: u32 = 0; + #[derive(Clone)] pub struct DojoWorld { world_address: FieldElement, @@ -116,9 +119,20 @@ impl DojoWorld { async fn entities_by_attribute( &self, attribute: proto::types::AttributeClause, + limit_offset: Option, ) -> Result, Error> { + let (limit, offset) = match limit_offset { + Some(values) => (values.limit, values.offset), + None => (DEFAULT_LIMIT, DEFAULT_OFFSET), + }; + let schema = self.model_cache.schema(&attribute.model).await?; - let rows = sqlx::query(&schema.sql).fetch_all(&self.pool).await?; + let rows = sqlx::query(&format!("{} LIMIT ? OFFSET ?", schema.sql)) + .bind(limit) + .bind(offset) + .fetch_all(&self.pool) + .await?; + let models = self.map_rows_to_models(&schema.ty, &rows).await?; Ok(vec![proto::types::Entity { key: "".to_string(), models }]) @@ -195,7 +209,9 @@ impl DojoWorld { let entities = match clause_type { ClauseType::Keys(keys) => self.entities_by_keys(keys).await?, - ClauseType::Attribute(attribute) => self.entities_by_attribute(attribute).await?, + ClauseType::Attribute(attribute) => { + self.entities_by_attribute(attribute, query.limit_offset).await? + } ClauseType::Composite(composite) => self.entities_by_composite(composite).await?, }; diff --git a/crates/torii/grpc/src/types.rs b/crates/torii/grpc/src/types.rs index ef147bad36..122b0606be 100644 --- a/crates/torii/grpc/src/types.rs +++ b/crates/torii/grpc/src/types.rs @@ -13,6 +13,13 @@ use crate::proto; #[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] pub struct Query { pub clause: Clause, + pub offset_limit: LimitOffset, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] +pub struct LimitOffset { + pub offset: u32, + pub limit: u32, } #[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] @@ -105,7 +112,13 @@ impl TryFrom for dojo_types::WorldMetadata { impl From for proto::types::EntityQuery { fn from(value: Query) -> Self { - Self { clause: Some(value.clause.into()) } + Self { clause: Some(value.clause.into()), limit_offset: Some(value.offset_limit.into()) } + } +} + +impl From for proto::types::LimitOffset { + fn from(value: LimitOffset) -> Self { + Self { limit: value.limit, offset: value.offset } } } From c40528403a9dc36b28c0a800f621b62ff41a8d85 Mon Sep 17 00:00:00 2001 From: broody Date: Tue, 21 Nov 2023 16:03:00 -0800 Subject: [PATCH 12/16] refactor query logic --- crates/torii/core/src/cache.rs | 50 ++-- crates/torii/core/src/error.rs | 2 + crates/torii/core/src/model.rs | 229 ++++++++-------- .../src/tests/types-test/src/contracts.cairo | 5 +- .../src/tests/types-test/src/models.cairo | 7 + crates/torii/grpc/proto/types.proto | 2 +- crates/torii/grpc/src/server/mod.rs | 249 ++++++++++-------- scripts/rust_fmt.sh | 2 +- 8 files changed, 302 insertions(+), 244 deletions(-) diff --git a/crates/torii/core/src/cache.rs b/crates/torii/core/src/cache.rs index 0296b46d25..0eb7562c21 100644 --- a/crates/torii/core/src/cache.rs +++ b/crates/torii/core/src/cache.rs @@ -1,40 +1,49 @@ use std::collections::HashMap; -use std::sync::Arc; use dojo_types::schema::Ty; use sqlx::SqlitePool; use tokio::sync::RwLock; use crate::error::{Error, QueryError}; -use crate::model::{build_sql_model_query, parse_sql_model_members, SqlModelMember}; +use crate::model::{parse_sql_model_members, SqlModelMember}; -pub struct SchemaData { - pub ty: Ty, - pub sql: String, -} +type EntityId = String; +type ModelName = String; pub struct ModelCache { pool: SqlitePool, - schemas: RwLock>>, + models: RwLock>>, + schemas: RwLock>, } impl ModelCache { pub fn new(pool: SqlitePool) -> Self { - Self { pool, schemas: RwLock::new(HashMap::new()) } + Self { pool, models: RwLock::new(HashMap::new()), schemas: RwLock::new(HashMap::new()) } } - pub async fn schema(&self, model: &str) -> Result, Error> { + pub async fn model(&self, entity_id: &str) -> Result, Error> { + { + let models = self.models.read().await; + if let Some(models) = models.get(entity_id) { + return Ok(models.clone()); + } + } + + self.update_model(entity_id).await + } + + pub async fn schema(&self, model: &str) -> Result { { let schemas = self.schemas.read().await; if let Some(schema) = schemas.get(model) { - return Ok(Arc::clone(schema)); + return Ok(schema.clone()); } } - self.update(model).await + self.update_schema(model).await } - async fn update(&self, model: &str) -> Result, Error> { + async fn update_schema(&self, model: &str) -> Result { let model_members: Vec = sqlx::query_as( "SELECT id, model_idx, member_idx, name, type, type_enum, enum_options, key FROM \ model_members WHERE model_id = ? ORDER BY model_idx ASC, member_idx ASC", @@ -48,13 +57,20 @@ impl ModelCache { } let ty = parse_sql_model_members(model, &model_members); - let sql = build_sql_model_query(ty.as_struct().unwrap()); - let schema = Arc::new(SchemaData { ty, sql }); - let mut schemas = self.schemas.write().await; - schemas.insert(model.into(), Arc::clone(&schema)); + schemas.insert(model.into(), ty.clone()); + + Ok(ty) + } + + async fn update_model(&self, entity_id: &str) -> Result, Error> { + let (model_names,): (String,) = + sqlx::query_as("SELECT model_names FROM entities WHERE id = ?") + .bind(entity_id) + .fetch_one(&self.pool) + .await?; - Ok(schema) + Ok(model_names.split(",").map(|s| s.to_string()).collect()) } pub async fn clear(&self) { diff --git a/crates/torii/core/src/error.rs b/crates/torii/core/src/error.rs index 1cb4061559..0d73633076 100644 --- a/crates/torii/core/src/error.rs +++ b/crates/torii/core/src/error.rs @@ -37,4 +37,6 @@ pub enum QueryError { UnsupportedQuery, #[error("model not found: {0}")] ModelNotFound(String), + #[error("exceeds sqlite `JOIN` limit (64)")] + SqliteJoinLimit, } diff --git a/crates/torii/core/src/model.rs b/crates/torii/core/src/model.rs index 433cd29cb8..79da329a0c 100644 --- a/crates/torii/core/src/model.rs +++ b/crates/torii/core/src/model.rs @@ -10,7 +10,7 @@ use sqlx::{Pool, Row, Sqlite}; use starknet::core::types::FieldElement; use super::error::{self, Error}; -use crate::error::ParseError; +use crate::error::{ParseError, QueryError}; pub struct ModelSQLReader { /// The name of the model @@ -150,9 +150,9 @@ pub fn parse_sql_model_members(model: &str, model_members_all: &[SqlModelMember] parse_sql_model_members_impl(model, model_members_all) } -/// A helper function to build a model query including all nested structs and its the entity id -pub fn build_sql_model_query(schema: &Struct) -> String { - fn build_sql_model_query_impl( +/// Creates a query that fetches all models and their nested structures. +pub fn build_sql_query(model_schemas: &Vec) -> Result { + fn parse_struct( path: &str, schema: &Struct, selections: &mut Vec, @@ -162,12 +162,12 @@ pub fn build_sql_model_query(schema: &Struct) -> String { match &child.ty { Ty::Struct(s) => { let table_name = format!("{}${}", path, s.name); - build_sql_model_query_impl(&table_name, s, selections, tables); + parse_struct(&table_name, s, selections, tables); tables.push(table_name); } _ => { - // alias selected columns to avoid conflicts in JOIN + // alias selected columns to avoid conflicts in `JOIN` selections.push(format!( "{}.external_{} AS \"{}.{}\"", path, child.name, path, child.name @@ -177,127 +177,127 @@ pub fn build_sql_model_query(schema: &Struct) -> String { } } - let model_table = &schema.name; - let mut selections = Vec::new(); - let mut tables = Vec::new(); + let primary_table = model_schemas[0].name(); + let mut global_selections = Vec::new(); + let mut global_tables = model_schemas + .iter() + .enumerate() + .filter(|(index, _)| *index != 0) // primary_table don't `JOIN` itself + .map(|(_, schema)| schema.name()) + .collect::>(); + + for ty in model_schemas { + let schema = ty.as_struct().expect("schema should be struct"); + let model_table = &schema.name; + let mut selections = Vec::new(); + let mut tables = Vec::new(); + + parse_struct(model_table, schema, &mut selections, &mut tables); + + global_selections.push(selections.join(", ")); + global_tables.extend(tables); + } + + // TODO: Fallback to subqueries, SQLite has a max limit of 64 on 'table 'JOIN' + if global_tables.len() > 64 { + return Err(QueryError::SqliteJoinLimit.into()); + } - build_sql_model_query_impl(model_table, schema, &mut selections, &mut tables); - let selections_clause = selections.join(", "); - let join_clause = tables + let selections_clause = global_selections.join(", "); + let join_clause = global_tables .into_iter() - .map(|table| { - format!(" LEFT JOIN {} ON {}.entity_id = {}.entity_id", table, model_table, table) - }) + .map(|table| format!(" LEFT JOIN {table} ON {primary_table}.entity_id = {table}.entity_id")) .collect::>() .join(" "); - format!( - "SELECT {selections_clause} FROM {model_table}{join_clause} ORDER BY \ - {model_table}.event_id" - ) + Ok(format!("SELECT {selections_clause} FROM {primary_table}{join_clause}")) } -/// Converts SQLite rows into a vector of `Ty` based on a specified schema. -pub fn map_rows_to_tys(schema: &Ty, rows: &[SqliteRow]) -> Result, Error> { - fn populate_struct_from_row( - path: &str, - struct_ty: &mut Struct, - row: &SqliteRow, - ) -> Result<(), Error> { - for member in struct_ty.children.iter_mut() { - let column_name = format!("{}.{}", path, member.name); - match &mut member.ty { - Ty::Primitive(primitive) => { - match &primitive { - Primitive::Bool(_) => { - let value = row.try_get::(&column_name)?; - primitive.set_bool(Some(value))?; - } - Primitive::USize(_) => { - let value = row.try_get::(&column_name)?; - primitive.set_usize(Some(value))?; - } - Primitive::U8(_) => { - let value = row.try_get::(&column_name)?; - primitive.set_u8(Some(value))?; - } - Primitive::U16(_) => { - let value = row.try_get::(&column_name)?; - primitive.set_u16(Some(value))?; - } - Primitive::U32(_) => { - let value = row.try_get::(&column_name)?; - primitive.set_u32(Some(value))?; - } - Primitive::U64(_) => { - let value = row.try_get::(&column_name)?; - primitive.set_u64(Some(value as u64))?; - } - Primitive::U128(_) => { - let value = row.try_get::(&column_name)?; - let hex_str = value.trim_start_matches("0x"); - primitive.set_u128(Some( - u128::from_str_radix(hex_str, 16) - .map_err(ParseError::ParseIntError)?, - ))?; - } - Primitive::U256(_) => { - let value = row.try_get::(&column_name)?; - let hex_str = value.trim_start_matches("0x"); - primitive.set_u256(Some(U256::from_be_hex(hex_str)))?; - } - Primitive::Felt252(_) => { - let value = row.try_get::(&column_name)?; - primitive.set_felt252(Some( - FieldElement::from_str(&value).map_err(ParseError::FromStr)?, - ))?; - } - Primitive::ClassHash(_) => { - let value = row.try_get::(&column_name)?; - primitive.set_class_hash(Some( - FieldElement::from_str(&value).map_err(ParseError::FromStr)?, - ))?; - } - Primitive::ContractAddress(_) => { - let value = row.try_get::(&column_name)?; - primitive.set_contract_address(Some( - FieldElement::from_str(&value).map_err(ParseError::FromStr)?, - ))?; - } - }; - } - Ty::Enum(enum_ty) => { - let value = row.try_get::(&column_name)?; - enum_ty.set_option(&value)?; - } - Ty::Struct(struct_ty) => { - let path = [path, &struct_ty.name].join("$"); - populate_struct_from_row(&path, struct_ty, row)?; - } - ty => { - unimplemented!("unimplemented type_enum: {ty}"); - } - }; - } - - Ok(()) +/// Populate the values of a Ty (schema) from SQLite row. +pub fn map_row_to_ty(path: &str, struct_ty: &mut Struct, row: &SqliteRow) -> Result<(), Error> { + for member in struct_ty.children.iter_mut() { + let column_name = format!("{}.{}", path, member.name); + match &mut member.ty { + Ty::Primitive(primitive) => { + match &primitive { + Primitive::Bool(_) => { + let value = row.try_get::(&column_name)?; + primitive.set_bool(Some(value))?; + } + Primitive::USize(_) => { + let value = row.try_get::(&column_name)?; + primitive.set_usize(Some(value))?; + } + Primitive::U8(_) => { + let value = row.try_get::(&column_name)?; + primitive.set_u8(Some(value))?; + } + Primitive::U16(_) => { + let value = row.try_get::(&column_name)?; + primitive.set_u16(Some(value))?; + } + Primitive::U32(_) => { + let value = row.try_get::(&column_name)?; + primitive.set_u32(Some(value))?; + } + Primitive::U64(_) => { + let value = row.try_get::(&column_name)?; + primitive.set_u64(Some(value as u64))?; + } + Primitive::U128(_) => { + let value = row.try_get::(&column_name)?; + let hex_str = value.trim_start_matches("0x"); + primitive.set_u128(Some( + u128::from_str_radix(hex_str, 16).map_err(ParseError::ParseIntError)?, + ))?; + } + Primitive::U256(_) => { + let value = row.try_get::(&column_name)?; + let hex_str = value.trim_start_matches("0x"); + primitive.set_u256(Some(U256::from_be_hex(hex_str)))?; + } + Primitive::Felt252(_) => { + let value = row.try_get::(&column_name)?; + primitive.set_felt252(Some( + FieldElement::from_str(&value).map_err(ParseError::FromStr)?, + ))?; + } + Primitive::ClassHash(_) => { + let value = row.try_get::(&column_name)?; + primitive.set_class_hash(Some( + FieldElement::from_str(&value).map_err(ParseError::FromStr)?, + ))?; + } + Primitive::ContractAddress(_) => { + let value = row.try_get::(&column_name)?; + primitive.set_contract_address(Some( + FieldElement::from_str(&value).map_err(ParseError::FromStr)?, + ))?; + } + }; + } + Ty::Enum(enum_ty) => { + let value = row.try_get::(&column_name)?; + enum_ty.set_option(&value)?; + } + Ty::Struct(struct_ty) => { + let path = [path, &struct_ty.name].join("$"); + map_row_to_ty(&path, struct_ty, row)?; + } + ty => { + unimplemented!("unimplemented type_enum: {ty}"); + } + }; } - rows.iter() - .map(|row| { - let mut struct_ty = schema.as_struct().expect("schema should be struct ty").clone(); - populate_struct_from_row(&schema.name(), &mut struct_ty, row)?; - - Ok(Ty::Struct(struct_ty)) - }) - .collect::, Error>>() + Ok(()) } #[cfg(test)] mod tests { use dojo_types::schema::{Enum, EnumOption, Member, Struct, Ty}; - use super::{build_sql_model_query, SqlModelMember}; + use super::{build_sql_query, SqlModelMember}; use crate::model::parse_sql_model_members; #[test] @@ -508,14 +508,13 @@ mod tests { ], }); - let query = build_sql_model_query(ty.as_struct().unwrap()); + let query = build_sql_query(&vec![ty.clone()]).unwrap(); assert_eq!( query, "SELECT Position.external_name AS \"Position.name\", Position.external_age AS \ \"Position.age\", Position$Vec2.external_x AS \"Position$Vec2.x\", \ Position$Vec2.external_y AS \"Position$Vec2.y\" FROM Position LEFT JOIN \ - Position$Vec2 ON Position.entity_id = Position$Vec2.entity_id ORDER BY \ - Position.event_id" + Position$Vec2 ON Position.entity_id = Position$Vec2.entity_id" ); } } diff --git a/crates/torii/graphql/src/tests/types-test/src/contracts.cairo b/crates/torii/graphql/src/tests/types-test/src/contracts.cairo index 1e5a0a3dca..638df904a6 100644 --- a/crates/torii/graphql/src/tests/types-test/src/contracts.cairo +++ b/crates/torii/graphql/src/tests/types-test/src/contracts.cairo @@ -8,7 +8,7 @@ trait IRecords { #[dojo::contract] mod records { use starknet::{ContractAddress, get_caller_address}; - use types_test::models::{Record, Subrecord, Nested, NestedMore, NestedMoreMore, Depth}; + use types_test::models::{Record, RecordSibling, Subrecord, Nested, NestedMore, NestedMoreMore, Depth}; use types_test::{seed, random}; use super::IRecords; @@ -90,6 +90,9 @@ mod records { random_u8, random_u128 }, + RecordSibling { + record_id, random_u8 + }, Subrecord { record_id, subrecord_id, type_u8: record_idx.into(), random_u8, } diff --git a/crates/torii/graphql/src/tests/types-test/src/models.cairo b/crates/torii/graphql/src/tests/types-test/src/models.cairo index af855773d4..a68b11ae8e 100644 --- a/crates/torii/graphql/src/tests/types-test/src/models.cairo +++ b/crates/torii/graphql/src/tests/types-test/src/models.cairo @@ -21,6 +21,13 @@ struct Record { random_u128: u128, } +#[derive(Model, Copy, Drop, Serde)] +struct RecordSibling { + #[key] + record_id: u32, + random_u8: u8 +} + #[derive(Copy, Drop, Serde, Introspect)] struct Nested { depth: Depth, diff --git a/crates/torii/grpc/proto/types.proto b/crates/torii/grpc/proto/types.proto index 101acdfd0d..ffed1331a8 100644 --- a/crates/torii/grpc/proto/types.proto +++ b/crates/torii/grpc/proto/types.proto @@ -56,7 +56,7 @@ message Model { message Entity { // The entity key - string key = 1; + bytes key = 1; // Models of the entity repeated Model models = 2; } diff --git a/crates/torii/grpc/src/server/mod.rs b/crates/torii/grpc/src/server/mod.rs index 8792ec525f..0f4582b27a 100644 --- a/crates/torii/grpc/src/server/mod.rs +++ b/crates/torii/grpc/src/server/mod.rs @@ -5,6 +5,7 @@ pub mod subscription; use std::future::Future; use std::net::SocketAddr; use std::pin::Pin; +use std::str::FromStr; use std::sync::Arc; use dojo_types::primitive::Primitive; @@ -27,6 +28,7 @@ use tonic::transport::Server; use tonic::{Request, Response, Status}; use torii_core::cache::ModelCache; use torii_core::error::{Error, ParseError, QueryError}; +use torii_core::model::build_sql_query; use self::subscription::SubscribeRequest; use crate::proto::types::clause::ClauseType; @@ -96,7 +98,7 @@ impl DojoWorld { packed_size: model.2, unpacked_size: model.3, layout: hex::decode(&model.4).unwrap(), - schema: serde_json::to_vec(&schema.ty).unwrap(), + schema: serde_json::to_vec(&schema).unwrap(), }); } @@ -111,38 +113,78 @@ impl DojoWorld { async fn entities_by_keys( &self, - _keys: proto::types::KeysClause, + keys_clause: proto::types::KeysClause, + limit_offset: Option, ) -> Result, Error> { - Ok(vec![]) + let keys = keys_clause + .keys + .iter() + .map(|bytes| { + if bytes.is_empty() { + return Ok("%".to_string()); + } + Ok(FieldElement::from_byte_slice_be(&bytes) + .map(|felt| format!("{:#x}", felt)) + .map_err(ParseError::FromByteSliceError)?) + }) + .collect::, Error>>()?; + let keys_pattern = keys.join("/") + "/%"; + + let (limit, offset) = limit_offset + .map(|values| (values.limit, values.offset)) + .unwrap_or((DEFAULT_LIMIT, DEFAULT_OFFSET)); + + let db_entities: Vec<(String, String)> = sqlx::query_as( + "SELECT id, model_names FROM entities WHERE keys LIKE ? ORDER BY event_id ASC LIMIT ? \ + OFFSET ?", + ) + .bind(&keys_pattern) + .bind(&limit) + .bind(&offset) + .fetch_all(&self.pool) + .await?; + + let mut entities = Vec::new(); + for (entity_id, models_str) in db_entities { + let model_names: Vec<&str> = models_str.split(",").collect(); + let mut schemas = Vec::new(); + for model in &model_names { + schemas.push(self.model_cache.schema(model).await?); + } + + let entity_query = + format!("{} WHERE {}.entity_id = ?", build_sql_query(&schemas)?, schemas[0].name()); + let row = sqlx::query(&entity_query).bind(&entity_id).fetch_one(&self.pool).await?; + + let mut models = Vec::new(); + for schema in schemas { + let struct_ty = schema.as_struct().expect("schema should be struct"); + models.push(self.map_row_to_model(&schema.name(), struct_ty, &row)?); + } + + let key = FieldElement::from_str(&entity_id).map_err(ParseError::FromStr)?; + entities.push(proto::types::Entity { key: key.to_bytes_be().to_vec(), models }) + } + + Ok(entities) } async fn entities_by_attribute( &self, - attribute: proto::types::AttributeClause, - limit_offset: Option, + _attribute: proto::types::AttributeClause, + _limit_offset: Option, ) -> Result, Error> { - let (limit, offset) = match limit_offset { - Some(values) => (values.limit, values.offset), - None => (DEFAULT_LIMIT, DEFAULT_OFFSET), - }; - - let schema = self.model_cache.schema(&attribute.model).await?; - let rows = sqlx::query(&format!("{} LIMIT ? OFFSET ?", schema.sql)) - .bind(limit) - .bind(offset) - .fetch_all(&self.pool) - .await?; - - let models = self.map_rows_to_models(&schema.ty, &rows).await?; - - Ok(vec![proto::types::Entity { key: "".to_string(), models }]) + // TODO: Implement + Err(QueryError::UnsupportedQuery.into()) } async fn entities_by_composite( &self, _composite: proto::types::CompositeClause, + _limit_offset: Option, ) -> Result, Error> { - Ok(vec![]) + // TODO: Implement + Err(QueryError::UnsupportedQuery.into()) } pub async fn model_metadata(&self, model: &str) -> Result { @@ -168,7 +210,7 @@ impl DojoWorld { class_hash, packed_size, unpacked_size, - schema: serde_json::to_vec(&schema.ty).unwrap(), + schema: serde_json::to_vec(&schema).unwrap(), }) } @@ -208,109 +250,98 @@ impl DojoWorld { .ok_or(QueryError::UnsupportedQuery)?; let entities = match clause_type { - ClauseType::Keys(keys) => self.entities_by_keys(keys).await?, + ClauseType::Keys(keys) => self.entities_by_keys(keys, query.limit_offset).await?, ClauseType::Attribute(attribute) => { self.entities_by_attribute(attribute, query.limit_offset).await? } - ClauseType::Composite(composite) => self.entities_by_composite(composite).await?, + ClauseType::Composite(composite) => { + self.entities_by_composite(composite, query.limit_offset).await? + } }; Ok(RetrieveEntitiesResponse { entities }) } - async fn map_rows_to_models( + fn map_row_to_model( &self, - schema: &Ty, - rows: &[SqliteRow], - ) -> Result, Error> { - fn row_to_model( - path: &str, - struct_ty: &Struct, - row: &SqliteRow, - ) -> Result { - let members = struct_ty - .children - .iter() - .map(|member| { - let column_name = format!("{}.{}", path, member.name); - let name = member.name.clone(); - let member = match &member.ty { - Ty::Primitive(primitive) => { - let value_type = match primitive { - Primitive::Bool(_) => proto::types::value::ValueType::BoolValue( - row.try_get::(&column_name)?, - ), - Primitive::U8(_) - | Primitive::U16(_) - | Primitive::U32(_) - | Primitive::U64(_) - | Primitive::USize(_) => { - let value = row.try_get::(&column_name)?; - proto::types::value::ValueType::UintValue(value as u64) - } - Primitive::U128(_) - | Primitive::U256(_) - | Primitive::Felt252(_) - | Primitive::ClassHash(_) - | Primitive::ContractAddress(_) => { - let value = row.try_get::(&column_name)?; - proto::types::value::ValueType::StringValue(value) - } - }; - - proto::types::Member { - name, - member_type: Some(proto::types::member::MemberType::Value( - proto::types::Value { value_type: Some(value_type) }, - )), + path: &str, + struct_ty: &Struct, + row: &SqliteRow, + ) -> Result { + let members = struct_ty + .children + .iter() + .map(|member| { + let column_name = format!("{}.{}", path, member.name); + let name = member.name.clone(); + let member = match &member.ty { + Ty::Primitive(primitive) => { + let value_type = match primitive { + Primitive::Bool(_) => proto::types::value::ValueType::BoolValue( + row.try_get::(&column_name)?, + ), + Primitive::U8(_) + | Primitive::U16(_) + | Primitive::U32(_) + | Primitive::U64(_) + | Primitive::USize(_) => { + let value = row.try_get::(&column_name)?; + proto::types::value::ValueType::UintValue(value as u64) } - } - - Ty::Enum(enum_ty) => { - let value = row.try_get::(&column_name)?; - let options = enum_ty - .options - .iter() - .map(|e| e.name.to_string()) - .collect::>(); - let option = - options.iter().position(|o| o == &value).expect("wrong enum value") - as u32; - proto::types::Member { - name: enum_ty.name.clone(), - member_type: Some(proto::types::member::MemberType::Enum( - proto::types::Enum { option, options }, - )), + Primitive::U128(_) + | Primitive::U256(_) + | Primitive::Felt252(_) + | Primitive::ClassHash(_) + | Primitive::ContractAddress(_) => { + let value = row.try_get::(&column_name)?; + proto::types::value::ValueType::StringValue(value) } + }; + + proto::types::Member { + name, + member_type: Some(proto::types::member::MemberType::Value( + proto::types::Value { value_type: Some(value_type) }, + )), } - Ty::Struct(struct_ty) => { - let path = [path, &struct_ty.name].join("$"); - proto::types::Member { - name, - member_type: Some(proto::types::member::MemberType::Struct( - row_to_model(&path, struct_ty, row)?, - )), - } + } + + Ty::Enum(enum_ty) => { + let value = row.try_get::(&column_name)?; + let options = enum_ty + .options + .iter() + .map(|e| e.name.to_string()) + .collect::>(); + let option = + options.iter().position(|o| o == &value).expect("wrong enum value") + as u32; + proto::types::Member { + name: enum_ty.name.clone(), + member_type: Some(proto::types::member::MemberType::Enum( + proto::types::Enum { option, options }, + )), } - ty => { - unimplemented!("unimplemented type_enum: {ty}"); + } + Ty::Struct(struct_ty) => { + let path = [path, &struct_ty.name].join("$"); + proto::types::Member { + name, + member_type: Some(proto::types::member::MemberType::Struct( + self.map_row_to_model(&path, struct_ty, row)?, + )), } - }; + } + ty => { + unimplemented!("unimplemented type_enum: {ty}"); + } + }; - Ok(member) - }) - .collect::, Error>>()?; - - Ok(proto::types::Model { name: struct_ty.name.clone(), members }) - } - - rows.iter() - .map(|row| { - let struct_ty = schema.as_struct().expect("schema should be struct ty").clone(); - - row_to_model(&schema.name(), &struct_ty, row) + Ok(member) }) - .collect::, Error>>() + .collect::, Error>>()?; + + Ok(proto::types::Model { name: struct_ty.name.clone(), members }) } } diff --git a/scripts/rust_fmt.sh b/scripts/rust_fmt.sh index 62a418693a..095c105201 100755 --- a/scripts/rust_fmt.sh +++ b/scripts/rust_fmt.sh @@ -1,3 +1,3 @@ #!/bin/bash -cargo +nightly fmt --check --all -- "$@" +cargo +nightly fmt --all -- "$@" From 9765aee956bf5a2e514a44c035068e351cd3d055 Mon Sep 17 00:00:00 2001 From: broody Date: Tue, 21 Nov 2023 17:09:13 -0800 Subject: [PATCH 13/16] clippy & fmt --- crates/torii/core/src/cache.rs | 2 +- crates/torii/grpc/src/server/mod.rs | 14 ++++++-------- scripts/rust_fmt.sh | 2 +- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/crates/torii/core/src/cache.rs b/crates/torii/core/src/cache.rs index 0eb7562c21..3bdb67aba1 100644 --- a/crates/torii/core/src/cache.rs +++ b/crates/torii/core/src/cache.rs @@ -70,7 +70,7 @@ impl ModelCache { .fetch_one(&self.pool) .await?; - Ok(model_names.split(",").map(|s| s.to_string()).collect()) + Ok(model_names.split(',').map(|s| s.to_string()).collect()) } pub async fn clear(&self) { diff --git a/crates/torii/grpc/src/server/mod.rs b/crates/torii/grpc/src/server/mod.rs index 0f4582b27a..207557b92f 100644 --- a/crates/torii/grpc/src/server/mod.rs +++ b/crates/torii/grpc/src/server/mod.rs @@ -123,7 +123,7 @@ impl DojoWorld { if bytes.is_empty() { return Ok("%".to_string()); } - Ok(FieldElement::from_byte_slice_be(&bytes) + Ok(FieldElement::from_byte_slice_be(bytes) .map(|felt| format!("{:#x}", felt)) .map_err(ParseError::FromByteSliceError)?) }) @@ -139,14 +139,14 @@ impl DojoWorld { OFFSET ?", ) .bind(&keys_pattern) - .bind(&limit) - .bind(&offset) + .bind(limit) + .bind(offset) .fetch_all(&self.pool) .await?; let mut entities = Vec::new(); for (entity_id, models_str) in db_entities { - let model_names: Vec<&str> = models_str.split(",").collect(); + let model_names: Vec<&str> = models_str.split(',').collect(); let mut schemas = Vec::new(); for model in &model_names { schemas.push(self.model_cache.schema(model).await?); @@ -159,7 +159,7 @@ impl DojoWorld { let mut models = Vec::new(); for schema in schemas { let struct_ty = schema.as_struct().expect("schema should be struct"); - models.push(self.map_row_to_model(&schema.name(), struct_ty, &row)?); + models.push(Self::map_row_to_model(&schema.name(), struct_ty, &row)?); } let key = FieldElement::from_str(&entity_id).map_err(ParseError::FromStr)?; @@ -263,7 +263,6 @@ impl DojoWorld { } fn map_row_to_model( - &self, path: &str, struct_ty: &Struct, row: &SqliteRow, @@ -305,7 +304,6 @@ impl DojoWorld { )), } } - Ty::Enum(enum_ty) => { let value = row.try_get::(&column_name)?; let options = enum_ty @@ -328,7 +326,7 @@ impl DojoWorld { proto::types::Member { name, member_type: Some(proto::types::member::MemberType::Struct( - self.map_row_to_model(&path, struct_ty, row)?, + Self::map_row_to_model(&path, struct_ty, row)?, )), } } diff --git a/scripts/rust_fmt.sh b/scripts/rust_fmt.sh index 095c105201..62a418693a 100755 --- a/scripts/rust_fmt.sh +++ b/scripts/rust_fmt.sh @@ -1,3 +1,3 @@ #!/bin/bash -cargo +nightly fmt --all -- "$@" +cargo +nightly fmt --check --all -- "$@" From f3d8f5675be539313aea7d4a76c530af83d75a00 Mon Sep 17 00:00:00 2001 From: broody Date: Tue, 21 Nov 2023 17:12:15 -0800 Subject: [PATCH 14/16] remove unused --- crates/torii/core/src/cache.rs | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/crates/torii/core/src/cache.rs b/crates/torii/core/src/cache.rs index 3bdb67aba1..dc8d39bc63 100644 --- a/crates/torii/core/src/cache.rs +++ b/crates/torii/core/src/cache.rs @@ -7,29 +7,16 @@ use tokio::sync::RwLock; use crate::error::{Error, QueryError}; use crate::model::{parse_sql_model_members, SqlModelMember}; -type EntityId = String; type ModelName = String; pub struct ModelCache { pool: SqlitePool, - models: RwLock>>, schemas: RwLock>, } impl ModelCache { pub fn new(pool: SqlitePool) -> Self { - Self { pool, models: RwLock::new(HashMap::new()), schemas: RwLock::new(HashMap::new()) } - } - - pub async fn model(&self, entity_id: &str) -> Result, Error> { - { - let models = self.models.read().await; - if let Some(models) = models.get(entity_id) { - return Ok(models.clone()); - } - } - - self.update_model(entity_id).await + Self { pool, schemas: RwLock::new(HashMap::new()) } } pub async fn schema(&self, model: &str) -> Result { @@ -63,16 +50,6 @@ impl ModelCache { Ok(ty) } - async fn update_model(&self, entity_id: &str) -> Result, Error> { - let (model_names,): (String,) = - sqlx::query_as("SELECT model_names FROM entities WHERE id = ?") - .bind(entity_id) - .fetch_one(&self.pool) - .await?; - - Ok(model_names.split(',').map(|s| s.to_string()).collect()) - } - pub async fn clear(&self) { self.schemas.write().await.clear(); } From beb6801bfb5d21e23adb6b9835796b892ad65167 Mon Sep 17 00:00:00 2001 From: broody Date: Tue, 21 Nov 2023 17:42:08 -0800 Subject: [PATCH 15/16] fix tests --- crates/dojo-types/src/primitive.rs | 3 ++- crates/torii/core/src/model.rs | 4 ++-- crates/torii/graphql/src/tests/entities_test.rs | 2 +- crates/torii/graphql/src/tests/models_test.rs | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/crates/dojo-types/src/primitive.rs b/crates/dojo-types/src/primitive.rs index 560dfc151c..2e91d0fdbb 100644 --- a/crates/dojo-types/src/primitive.rs +++ b/crates/dojo-types/src/primitive.rs @@ -65,7 +65,8 @@ macro_rules! set_primitive { /// Macro to generate getter methods for Primitive enum variants. macro_rules! as_primitive { ($method_name:ident, $variant:ident, $type:ty) => { - /// If the `Primitive` is type T, returns the associated [`T`]. Returns `None` otherwise. + /// If the `Primitive` is variant type, returns the associated vartiant value. Returns + /// `None` otherwise. pub fn $method_name(&self) -> Option<$type> { match self { Primitive::$variant(value) => *value, diff --git a/crates/torii/core/src/model.rs b/crates/torii/core/src/model.rs index 79da329a0c..c05831c664 100644 --- a/crates/torii/core/src/model.rs +++ b/crates/torii/core/src/model.rs @@ -150,7 +150,7 @@ pub fn parse_sql_model_members(model: &str, model_members_all: &[SqlModelMember] parse_sql_model_members_impl(model, model_members_all) } -/// Creates a query that fetches all models and their nested structures. +/// Creates a query that fetches all models and their nested data. pub fn build_sql_query(model_schemas: &Vec) -> Result { fn parse_struct( path: &str, @@ -508,7 +508,7 @@ mod tests { ], }); - let query = build_sql_query(&vec![ty.clone()]).unwrap(); + let query = build_sql_query(&vec![ty]).unwrap(); assert_eq!( query, "SELECT Position.external_name AS \"Position.name\", Position.external_age AS \ diff --git a/crates/torii/graphql/src/tests/entities_test.rs b/crates/torii/graphql/src/tests/entities_test.rs index a2119ba986..3f7c807828 100644 --- a/crates/torii/graphql/src/tests/entities_test.rs +++ b/crates/torii/graphql/src/tests/entities_test.rs @@ -92,7 +92,7 @@ mod tests { assert_eq!(connection.edges.len(), 10); assert_eq!(connection.total_count, 20); assert_eq!(&first_entity.node.model_names, "Subrecord"); - assert_eq!(&last_entity.node.model_names, "Record"); + assert_eq!(&last_entity.node.model_names, "Record,RecordSibling"); // first key param - returns all entities with `0x0` as first key let entities = entities_query(&schema, "(keys: [\"0x0\"])").await; diff --git a/crates/torii/graphql/src/tests/models_test.rs b/crates/torii/graphql/src/tests/models_test.rs index 0c6e3ca181..de28a4ea49 100644 --- a/crates/torii/graphql/src/tests/models_test.rs +++ b/crates/torii/graphql/src/tests/models_test.rs @@ -86,7 +86,7 @@ mod tests { assert_eq!(connection.total_count, 10); assert_eq!(connection.edges.len(), 10); assert_eq!(&record.node.__typename, "Record"); - assert_eq!(&entity.model_names, "Record"); + assert_eq!(&entity.model_names, "Record,RecordSibling"); assert_eq!(entity.keys.clone().unwrap(), vec!["0x0"]); assert_eq!(record.node.depth, "Zero"); assert_eq!(nested.depth, "One"); From b28560aeb626dc40aa73fad5aab8c8a1a9704934 Mon Sep 17 00:00:00 2001 From: broody Date: Tue, 21 Nov 2023 17:53:16 -0800 Subject: [PATCH 16/16] move limit offset to query clause --- crates/torii/grpc/proto/types.proto | 8 ++------ crates/torii/grpc/src/server/mod.rs | 24 +++++++++++------------- crates/torii/grpc/src/types.rs | 15 ++------------- 3 files changed, 15 insertions(+), 32 deletions(-) diff --git a/crates/torii/grpc/proto/types.proto b/crates/torii/grpc/proto/types.proto index ffed1331a8..80d9ae8da6 100644 --- a/crates/torii/grpc/proto/types.proto +++ b/crates/torii/grpc/proto/types.proto @@ -87,7 +87,8 @@ message EntityUpdate { message EntityQuery { Clause clause = 1; - LimitOffset limit_offset = 4; + uint32 limit = 2; + uint32 offset = 3; } message Clause { @@ -116,11 +117,6 @@ message CompositeClause { repeated Clause clauses = 3; } -message LimitOffset { - uint32 limit = 1; - uint32 offset = 2; -} - enum LogicalOperator { AND = 0; OR = 1; diff --git a/crates/torii/grpc/src/server/mod.rs b/crates/torii/grpc/src/server/mod.rs index 207557b92f..8b7d484817 100644 --- a/crates/torii/grpc/src/server/mod.rs +++ b/crates/torii/grpc/src/server/mod.rs @@ -35,9 +35,6 @@ use crate::proto::types::clause::ClauseType; use crate::proto::world::world_server::WorldServer; use crate::proto::{self}; -const DEFAULT_LIMIT: u32 = 10; -const DEFAULT_OFFSET: u32 = 0; - #[derive(Clone)] pub struct DojoWorld { world_address: FieldElement, @@ -114,7 +111,8 @@ impl DojoWorld { async fn entities_by_keys( &self, keys_clause: proto::types::KeysClause, - limit_offset: Option, + limit: u32, + offset: u32, ) -> Result, Error> { let keys = keys_clause .keys @@ -130,10 +128,6 @@ impl DojoWorld { .collect::, Error>>()?; let keys_pattern = keys.join("/") + "/%"; - let (limit, offset) = limit_offset - .map(|values| (values.limit, values.offset)) - .unwrap_or((DEFAULT_LIMIT, DEFAULT_OFFSET)); - let db_entities: Vec<(String, String)> = sqlx::query_as( "SELECT id, model_names FROM entities WHERE keys LIKE ? ORDER BY event_id ASC LIMIT ? \ OFFSET ?", @@ -172,7 +166,8 @@ impl DojoWorld { async fn entities_by_attribute( &self, _attribute: proto::types::AttributeClause, - _limit_offset: Option, + _limit: u32, + _offset: u32, ) -> Result, Error> { // TODO: Implement Err(QueryError::UnsupportedQuery.into()) @@ -181,7 +176,8 @@ impl DojoWorld { async fn entities_by_composite( &self, _composite: proto::types::CompositeClause, - _limit_offset: Option, + _limit: u32, + _offset: u32, ) -> Result, Error> { // TODO: Implement Err(QueryError::UnsupportedQuery.into()) @@ -250,12 +246,14 @@ impl DojoWorld { .ok_or(QueryError::UnsupportedQuery)?; let entities = match clause_type { - ClauseType::Keys(keys) => self.entities_by_keys(keys, query.limit_offset).await?, + ClauseType::Keys(keys) => { + self.entities_by_keys(keys, query.limit, query.offset).await? + } ClauseType::Attribute(attribute) => { - self.entities_by_attribute(attribute, query.limit_offset).await? + self.entities_by_attribute(attribute, query.limit, query.offset).await? } ClauseType::Composite(composite) => { - self.entities_by_composite(composite, query.limit_offset).await? + self.entities_by_composite(composite, query.limit, query.offset).await? } }; diff --git a/crates/torii/grpc/src/types.rs b/crates/torii/grpc/src/types.rs index 122b0606be..00efe50358 100644 --- a/crates/torii/grpc/src/types.rs +++ b/crates/torii/grpc/src/types.rs @@ -13,13 +13,8 @@ use crate::proto; #[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] pub struct Query { pub clause: Clause, - pub offset_limit: LimitOffset, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] -pub struct LimitOffset { - pub offset: u32, pub limit: u32, + pub offset: u32, } #[derive(Debug, Serialize, Deserialize, PartialEq, Hash, Eq, Clone)] @@ -112,13 +107,7 @@ impl TryFrom for dojo_types::WorldMetadata { impl From for proto::types::EntityQuery { fn from(value: Query) -> Self { - Self { clause: Some(value.clause.into()), limit_offset: Some(value.offset_limit.into()) } - } -} - -impl From for proto::types::LimitOffset { - fn from(value: LimitOffset) -> Self { - Self { limit: value.limit, offset: value.offset } + Self { clause: Some(value.clause.into()), limit: value.limit, offset: value.offset } } }