From b9bb4380adbbd7a14728725f5aa39e950e95abbb Mon Sep 17 00:00:00 2001 From: Yun Date: Wed, 4 Oct 2023 18:15:55 -0700 Subject: [PATCH] Torii indexer add id col to model tables (#979) * Torii indexer add id col to model tables * fix clippy too many arguments * fix tests --- crates/torii/core/src/engine.rs | 4 +- crates/torii/core/src/processors/mod.rs | 2 +- .../core/src/processors/register_model.rs | 4 +- .../core/src/processors/register_system.rs | 4 +- .../core/src/processors/store_set_record.rs | 8 +- crates/torii/core/src/sql.rs | 31 ++- crates/torii/graphql/src/tests/mod.rs | 252 ++++++++++-------- 7 files changed, 164 insertions(+), 141 deletions(-) diff --git a/crates/torii/core/src/engine.rs b/crates/torii/core/src/engine.rs index 1e5a7c1be7..f51afb2a78 100644 --- a/crates/torii/core/src/engine.rs +++ b/crates/torii/core/src/engine.rs @@ -232,7 +232,7 @@ async fn process_event( db: &mut Sql, provider: &P, processors: &[Box>], - block: &BlockWithTxs, + _block: &BlockWithTxs, invoke_receipt: &InvokeTransactionReceipt, event: &Event, event_idx: usize, @@ -241,7 +241,7 @@ async fn process_event( for processor in processors { if get_selector_from_name(&processor.event_key())? == event.keys[0] { - processor.process(world, db, provider, block, invoke_receipt, event).await?; + processor.process(world, db, provider, invoke_receipt, event, event_idx).await?; } } diff --git a/crates/torii/core/src/processors/mod.rs b/crates/torii/core/src/processors/mod.rs index 8a57e7f7a2..46efc8cecd 100644 --- a/crates/torii/core/src/processors/mod.rs +++ b/crates/torii/core/src/processors/mod.rs @@ -19,9 +19,9 @@ pub trait EventProcessor { world: &WorldContractReader<'_, P>, db: &mut Sql, provider: &P, - block: &BlockWithTxs, invoke_receipt: &InvokeTransactionReceipt, event: &Event, + event_idx: usize, ) -> Result<(), Error>; } diff --git a/crates/torii/core/src/processors/register_model.rs b/crates/torii/core/src/processors/register_model.rs index b9aca39fa9..a4d56ab31e 100644 --- a/crates/torii/core/src/processors/register_model.rs +++ b/crates/torii/core/src/processors/register_model.rs @@ -1,6 +1,6 @@ use anyhow::{Error, Ok, Result}; use async_trait::async_trait; -use starknet::core::types::{BlockId, BlockTag, BlockWithTxs, Event, InvokeTransactionReceipt}; +use starknet::core::types::{BlockId, BlockTag, Event, InvokeTransactionReceipt}; use starknet::core::utils::parse_cairo_short_string; use starknet::providers::Provider; use torii_client::contract::world::WorldContractReader; @@ -23,9 +23,9 @@ impl EventProcessor

for RegisterModelProcessor world: &WorldContractReader<'_, P>, db: &mut Sql, _provider: &P, - _block: &BlockWithTxs, _invoke_receipt: &InvokeTransactionReceipt, event: &Event, + _event_idx: usize, ) -> Result<(), Error> { let name = parse_cairo_short_string(&event.data[0])?; diff --git a/crates/torii/core/src/processors/register_system.rs b/crates/torii/core/src/processors/register_system.rs index e03be0ae44..2424c9894b 100644 --- a/crates/torii/core/src/processors/register_system.rs +++ b/crates/torii/core/src/processors/register_system.rs @@ -1,7 +1,7 @@ use anyhow::{Error, Ok, Result}; use async_trait::async_trait; use dojo_world::manifest::System; -use starknet::core::types::{BlockWithTxs, Event, InvokeTransactionReceipt}; +use starknet::core::types::{Event, InvokeTransactionReceipt}; use starknet::core::utils::parse_cairo_short_string; use starknet::providers::Provider; use torii_client::contract::world::WorldContractReader; @@ -24,9 +24,9 @@ impl EventProcessor

for RegisterSystemProcessor { _world: &WorldContractReader<'_, P>, db: &mut Sql, _provider: &P, - _block: &BlockWithTxs, _invoke_receipt: &InvokeTransactionReceipt, event: &Event, + _event_idx: usize, ) -> Result<(), Error> { let name = parse_cairo_short_string(&event.data[0])?; diff --git a/crates/torii/core/src/processors/store_set_record.rs b/crates/torii/core/src/processors/store_set_record.rs index 1f7ccce3f5..93e9a6b83a 100644 --- a/crates/torii/core/src/processors/store_set_record.rs +++ b/crates/torii/core/src/processors/store_set_record.rs @@ -1,6 +1,6 @@ use anyhow::{Error, Ok, Result}; use async_trait::async_trait; -use starknet::core::types::{BlockId, BlockTag, BlockWithTxs, Event, InvokeTransactionReceipt}; +use starknet::core::types::{BlockId, BlockTag, Event, InvokeTransactionReceipt}; use starknet::core::utils::parse_cairo_short_string; use starknet::providers::Provider; use starknet_crypto::FieldElement; @@ -27,9 +27,9 @@ impl EventProcessor

for StoreSetRecordProcessor world: &WorldContractReader<'_, P>, db: &mut Sql, _provider: &P, - _block: &BlockWithTxs, - _transaction_receipt: &InvokeTransactionReceipt, + transaction_receipt: &InvokeTransactionReceipt, event: &Event, + event_idx: usize, ) -> Result<(), Error> { let name = parse_cairo_short_string(&event.data[MODEL_INDEX])?; info!("store set record: {}", name); @@ -37,7 +37,7 @@ impl EventProcessor

for StoreSetRecordProcessor let model = world.model(&name, BlockId::Tag(BlockTag::Pending)).await?; let keys = values_at(&event.data, NUM_KEYS_INDEX)?; let entity = model.entity(keys, BlockId::Tag(BlockTag::Pending)).await?; - db.set_entity(entity).await?; + db.set_entity(entity, transaction_receipt.block_number, event_idx).await?; Ok(()) } } diff --git a/crates/torii/core/src/sql.rs b/crates/torii/core/src/sql.rs index cb3a2a16b7..cfa3194941 100644 --- a/crates/torii/core/src/sql.rs +++ b/crates/torii/core/src/sql.rs @@ -129,7 +129,7 @@ impl Sql { Ok(()) } - pub async fn set_entity(&mut self, entity: Ty) -> Result<()> { + pub async fn set_entity(&mut self, entity: Ty, block_num: u64, event_idx: usize) -> Result<()> { let keys = if let Ty::Struct(s) = &entity { let mut keys = Vec::new(); for m in s.keys() { @@ -162,7 +162,8 @@ impl Sql { )); let path = vec![entity.name()]; - self.build_set_entity_queries_recursive(path, &entity_id, &entity); + let id = format!("0x{:064x}:0x{:04x}", block_num, event_idx); + self.build_set_entity_queries_recursive(path, &id, &entity_id, &entity); self.execute().await?; @@ -222,7 +223,7 @@ impl Sql { let keys_str = felts_sql_string(&event.keys); let data_str = felts_sql_string(&event.data); - let id = format!("{:#x}:{}", transaction_hash, event_idx); + let id = format!("0x{:064x}:0x{:04x}", transaction_hash, event_idx); let query = format!( "INSERT OR IGNORE INTO events (id, keys, data, transaction_hash) VALUES ('{}', '{}', \ '{}', '{:#x}')", @@ -263,12 +264,18 @@ impl Sql { } } - fn build_set_entity_queries_recursive(&mut self, path: Vec, id: &str, entity: &Ty) { + fn build_set_entity_queries_recursive( + &mut self, + path: Vec, + id: &str, + entity_id: &str, + entity: &Ty, + ) { match entity { Ty::Struct(s) => { let table_id = path.join("$"); - let mut columns = vec!["entity_id".to_string()]; - let mut values = vec![format!("'{id}'")]; + let mut columns = vec!["id".to_string(), "entity_id".to_string()]; + let mut values = vec![format!("'{id}', '{entity_id}'")]; for member in s.children.iter() { match &member.ty { @@ -295,7 +302,9 @@ impl Sql { let mut path_clone = path.clone(); path_clone.push(member.ty.name()); - self.build_set_entity_queries_recursive(path_clone, id, &member.ty); + self.build_set_entity_queries_recursive( + path_clone, id, entity_id, &member.ty, + ); } } } @@ -304,7 +313,7 @@ impl Sql { let mut path_clone = path.clone(); path_clone.push(child.1.name()); // self.build_entity_query(path_clone.clone(), id, &child.1); - self.build_set_entity_queries_recursive(path_clone, id, &child.1); + self.build_set_entity_queries_recursive(path_clone, id, entity_id, &child.1); } } _ => {} @@ -315,7 +324,7 @@ impl Sql { let table_id = path.join("$"); let mut query = format!( - "CREATE TABLE IF NOT EXISTS [{table_id}] (entity_id TEXT NOT NULL PRIMARY KEY, " + "CREATE TABLE IF NOT EXISTS [{table_id}] (id TEXT NOT NULL PRIMARY KEY, entity_id, " ); if let Ty::Struct(s) = model { @@ -352,9 +361,7 @@ impl Sql { // If this is not the Model's root table, create a reference to the parent. if path.len() > 1 { let parent_table_id = path[..path.len() - 1].join("$"); - query.push_str(&format!( - "FOREIGN KEY (entity_id) REFERENCES {parent_table_id} (entity_id), " - )); + query.push_str(&format!("FOREIGN KEY (id) REFERENCES {parent_table_id} (id), ")); }; query.push_str("FOREIGN KEY (entity_id) REFERENCES entities(id));"); diff --git a/crates/torii/graphql/src/tests/mod.rs b/crates/torii/graphql/src/tests/mod.rs index 05d1b07cbc..6346d6386b 100644 --- a/crates/torii/graphql/src/tests/mod.rs +++ b/crates/torii/graphql/src/tests/mod.rs @@ -153,134 +153,150 @@ pub async fn entity_fixtures(db: &mut Sql) { .await .unwrap(); - db.set_entity(Ty::Struct(Struct { - name: "Moves".to_string(), - children: vec![ - Member { - name: "player".to_string(), - key: true, - ty: Ty::Primitive(Primitive::ContractAddress(Some(FieldElement::ONE))), - }, - Member { - name: "remaining".to_string(), - key: false, - ty: Ty::Primitive(Primitive::U8(Some(10))), - }, - Member { - name: "last_direction".to_string(), - key: false, - ty: Ty::Enum(Enum { - name: "Direction".to_string(), - option: Some(1), - options: vec![ - ("None".to_string(), Ty::Tuple(vec![])), - ("Left".to_string(), Ty::Tuple(vec![])), - ("Right".to_string(), Ty::Tuple(vec![])), - ("Up".to_string(), Ty::Tuple(vec![])), - ("Down".to_string(), Ty::Tuple(vec![])), - ], - }), - }, - ], - })) + db.set_entity( + Ty::Struct(Struct { + name: "Moves".to_string(), + children: vec![ + Member { + name: "player".to_string(), + key: true, + ty: Ty::Primitive(Primitive::ContractAddress(Some(FieldElement::ONE))), + }, + Member { + name: "remaining".to_string(), + key: false, + ty: Ty::Primitive(Primitive::U8(Some(10))), + }, + Member { + name: "last_direction".to_string(), + key: false, + ty: Ty::Enum(Enum { + name: "Direction".to_string(), + option: Some(1), + options: vec![ + ("None".to_string(), Ty::Tuple(vec![])), + ("Left".to_string(), Ty::Tuple(vec![])), + ("Right".to_string(), Ty::Tuple(vec![])), + ("Up".to_string(), Ty::Tuple(vec![])), + ("Down".to_string(), Ty::Tuple(vec![])), + ], + }), + }, + ], + }), + 0, + 0, + ) .await .unwrap(); - db.set_entity(Ty::Struct(Struct { - name: "Position".to_string(), - children: vec![ - Member { - name: "player".to_string(), - key: true, - ty: Ty::Primitive(Primitive::ContractAddress(Some(FieldElement::TWO))), - }, - Member { - name: "vec".to_string(), - key: false, - ty: Ty::Struct(Struct { - name: "Vec2".to_string(), - children: vec![ - Member { - name: "x".to_string(), - key: false, - ty: Ty::Primitive(Primitive::U32(Some(42))), - }, - Member { - name: "y".to_string(), - key: false, - ty: Ty::Primitive(Primitive::U32(Some(69))), - }, - ], - }), - }, - ], - })) + db.set_entity( + Ty::Struct(Struct { + name: "Position".to_string(), + children: vec![ + Member { + name: "player".to_string(), + key: true, + ty: Ty::Primitive(Primitive::ContractAddress(Some(FieldElement::TWO))), + }, + Member { + name: "vec".to_string(), + key: false, + ty: Ty::Struct(Struct { + name: "Vec2".to_string(), + children: vec![ + Member { + name: "x".to_string(), + key: false, + ty: Ty::Primitive(Primitive::U32(Some(42))), + }, + Member { + name: "y".to_string(), + key: false, + ty: Ty::Primitive(Primitive::U32(Some(69))), + }, + ], + }), + }, + ], + }), + 0, + 1, + ) .await .unwrap(); // Set an entity with both moves and position models - db.set_entity(Ty::Struct(Struct { - name: "Moves".to_string(), - children: vec![ - Member { - name: "player".to_string(), - key: true, - ty: Ty::Primitive(Primitive::ContractAddress(Some(FieldElement::THREE))), - }, - Member { - name: "remaining".to_string(), - key: false, - ty: Ty::Primitive(Primitive::U8(Some(10))), - }, - Member { - name: "last_direction".to_string(), - key: false, - ty: Ty::Enum(Enum { - name: "Direction".to_string(), - option: Some(2), - options: vec![ - ("None".to_string(), Ty::Tuple(vec![])), - ("Left".to_string(), Ty::Tuple(vec![])), - ("Right".to_string(), Ty::Tuple(vec![])), - ("Up".to_string(), Ty::Tuple(vec![])), - ("Down".to_string(), Ty::Tuple(vec![])), - ], - }), - }, - ], - })) + db.set_entity( + Ty::Struct(Struct { + name: "Moves".to_string(), + children: vec![ + Member { + name: "player".to_string(), + key: true, + ty: Ty::Primitive(Primitive::ContractAddress(Some(FieldElement::THREE))), + }, + Member { + name: "remaining".to_string(), + key: false, + ty: Ty::Primitive(Primitive::U8(Some(10))), + }, + Member { + name: "last_direction".to_string(), + key: false, + ty: Ty::Enum(Enum { + name: "Direction".to_string(), + option: Some(2), + options: vec![ + ("None".to_string(), Ty::Tuple(vec![])), + ("Left".to_string(), Ty::Tuple(vec![])), + ("Right".to_string(), Ty::Tuple(vec![])), + ("Up".to_string(), Ty::Tuple(vec![])), + ("Down".to_string(), Ty::Tuple(vec![])), + ], + }), + }, + ], + }), + 0, + 2, + ) .await .unwrap(); - db.set_entity(Ty::Struct(Struct { - name: "Position".to_string(), - children: vec![ - Member { - name: "player".to_string(), - key: true, - ty: Ty::Primitive(Primitive::ContractAddress(Some(FieldElement::THREE))), - }, - Member { - name: "vec".to_string(), - key: false, - ty: Ty::Struct(Struct { - name: "Vec2".to_string(), - children: vec![ - Member { - name: "x".to_string(), - key: false, - ty: Ty::Primitive(Primitive::U32(Some(42))), - }, - Member { - name: "y".to_string(), - key: false, - ty: Ty::Primitive(Primitive::U32(Some(69))), - }, - ], - }), - }, - ], - })) + db.set_entity( + Ty::Struct(Struct { + name: "Position".to_string(), + children: vec![ + Member { + name: "player".to_string(), + key: true, + ty: Ty::Primitive(Primitive::ContractAddress(Some(FieldElement::THREE))), + }, + Member { + name: "vec".to_string(), + key: false, + ty: Ty::Struct(Struct { + name: "Vec2".to_string(), + children: vec![ + Member { + name: "x".to_string(), + key: false, + ty: Ty::Primitive(Primitive::U32(Some(42))), + }, + Member { + name: "y".to_string(), + key: false, + ty: Ty::Primitive(Primitive::U32(Some(69))), + }, + ], + }), + }, + ], + }), + 0, + 3, + ) .await .unwrap();