From 1436ca703b03bfc2d5cade90f11cec0c9fecb9e8 Mon Sep 17 00:00:00 2001 From: lambda-0x <0xlambda@protonmail.com> Date: Mon, 19 Aug 2024 01:29:01 +0530 Subject: [PATCH] change table schema and core implementation --- .../src/processors/erc20_legacy_transfer.rs | 4 +- .../core/src/processors/erc20_transfer.rs | 4 +- .../core/src/processors/erc721_transfer.rs | 4 +- crates/torii/core/src/sql.rs | 243 +++++++++++++++--- crates/torii/core/src/types.rs | 10 + .../migrations/20240803102207_add_erc.sql | 80 ++---- 6 files changed, 256 insertions(+), 89 deletions(-) diff --git a/crates/torii/core/src/processors/erc20_legacy_transfer.rs b/crates/torii/core/src/processors/erc20_legacy_transfer.rs index ed5303049a..81044c700b 100644 --- a/crates/torii/core/src/processors/erc20_legacy_transfer.rs +++ b/crates/torii/core/src/processors/erc20_legacy_transfer.rs @@ -35,7 +35,7 @@ where async fn process( &self, - _world: &WorldContractReader

, + world: &WorldContractReader

, db: &mut Sql, _block_number: u64, _block_timestamp: u64, @@ -50,7 +50,7 @@ where let value = U256Cainome::cairo_deserialize(&event.data, 2)?; let value = U256::from_words(value.low, value.high); - db.handle_erc20_transfer(token_address, from, to, value).await?; + db.handle_erc20_transfer(token_address, from, to, value, world.provider()).await?; info!(target: LOG_TARGET,from = ?from, to = ?to, value = ?value, "Legacy ERC20 Transfer"); Ok(()) diff --git a/crates/torii/core/src/processors/erc20_transfer.rs b/crates/torii/core/src/processors/erc20_transfer.rs index 133fb74bd7..db7aaab0d7 100644 --- a/crates/torii/core/src/processors/erc20_transfer.rs +++ b/crates/torii/core/src/processors/erc20_transfer.rs @@ -35,7 +35,7 @@ where async fn process( &self, - _world: &WorldContractReader

, + world: &WorldContractReader

, db: &mut Sql, _block_number: u64, _block_timestamp: u64, @@ -50,7 +50,7 @@ where let value = U256Cainome::cairo_deserialize(&event.data, 0)?; let value = U256::from_words(value.low, value.high); - db.handle_erc20_transfer(token_address, from, to, value).await?; + db.handle_erc20_transfer(token_address, from, to, value, world.provider()).await?; info!(target: LOG_TARGET,from = ?from, to = ?to, value = ?value, "ERC20 Transfer"); Ok(()) diff --git a/crates/torii/core/src/processors/erc721_transfer.rs b/crates/torii/core/src/processors/erc721_transfer.rs index dc6f900c18..2faa09563d 100644 --- a/crates/torii/core/src/processors/erc721_transfer.rs +++ b/crates/torii/core/src/processors/erc721_transfer.rs @@ -36,7 +36,7 @@ where async fn process( &self, - _world: &WorldContractReader

, + world: &WorldContractReader

, db: &mut Sql, _block_number: u64, _block_timestamp: u64, @@ -51,7 +51,7 @@ where let token_id = U256Cainome::cairo_deserialize(&event.keys, 3)?; let token_id = U256::from_words(token_id.low, token_id.high); - db.handle_erc721_transfer(token_address, from, to, token_id).await?; + db.handle_erc721_transfer(token_address, from, to, token_id, world.provider()).await?; info!(target: LOG_TARGET, from = ?from, to = ?to, token_id = ?token_id, "ERC721 Transfer"); Ok(()) diff --git a/crates/torii/core/src/sql.rs b/crates/torii/core/src/sql.rs index b2e623ca2f..8d578c39ae 100644 --- a/crates/torii/core/src/sql.rs +++ b/crates/torii/core/src/sql.rs @@ -1,8 +1,10 @@ +use std::collections::HashMap; use std::convert::TryInto; use std::ops::{Add, Sub}; use std::str::FromStr; use anyhow::{anyhow, Result}; +use cainome::cairo_serde::{ByteArray, CairoSerde}; use chrono::Utc; use dojo_types::primitive::Primitive; use dojo_types::schema::{EnumOption, Member, Struct, Ty}; @@ -11,7 +13,12 @@ use dojo_world::contracts::naming::compute_selector_from_names; use dojo_world::metadata::WorldMetadata; use sqlx::pool::PoolConnection; use sqlx::{Pool, Row, Sqlite}; -use starknet::core::types::{Event, Felt, InvokeTransaction, Transaction, U256}; +use starknet::core::types::BlockTag; +use starknet::core::types::{ + BlockId, Event, Felt, FunctionCall, InvokeTransaction, Transaction, U256, +}; +use starknet::core::utils::{get_selector_from_name, parse_cairo_short_string}; +use starknet::providers::Provider; use starknet_crypto::poseidon_hash_many; use super::World; @@ -19,8 +26,8 @@ use crate::model::ModelSQLReader; use crate::query_queue::{Argument, QueryQueue}; use crate::simple_broker::SimpleBroker; use crate::types::{ - Entity as EntityUpdated, Event as EventEmitted, EventMessage as EventMessageUpdated, - Model as ModelRegistered, + Entity as EntityUpdated, ErcContract, Event as EventEmitted, + EventMessage as EventMessageUpdated, Model as ModelRegistered, }; use crate::utils::{must_utc_datetime_from_timestamp, utc_dt_string_from_timestamp}; @@ -45,6 +52,7 @@ impl Sql { pool: Pool, world_address: Felt, world_class_hash: Felt, + erc_contracts: &HashMap, ) -> Result { let mut query_queue = QueryQueue::new(pool.clone()); @@ -61,6 +69,17 @@ impl Sql { ], ); + for erc_contract in erc_contracts.values() { + query_queue.enqueue( + "INSERT OR IGNORE INTO contracts (id, contract_address, contract_type) VALUES (?, ?, ?)", + vec![ + Argument::FieldElement(erc_contract.contract_address), + Argument::FieldElement(erc_contract.contract_address), + Argument::String(erc_contract.r#type.to_string()), + ], + ); + } + query_queue.execute_all().await?; Ok(Self { pool, world_address, query_queue }) @@ -1185,13 +1204,89 @@ impl Sql { Ok(()) } - pub async fn handle_erc20_transfer( + pub async fn handle_erc20_transfer( &mut self, - token_address: Felt, + contract_address: Felt, from: Felt, to: Felt, amount: U256, + provider: &P, ) -> Result<()> { + let token_exists: bool = + sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM tokens WHERE id = ?)") + .bind(format!("{:#x}", contract_address)) + .fetch_one(&self.pool) + .await?; + + if !token_exists { + // Fetch token information from the chain + let name = provider + .call( + FunctionCall { + contract_address, + entry_point_selector: get_selector_from_name("name").unwrap(), + calldata: vec![], + }, + BlockId::Tag(BlockTag::Pending), + ) + .await?; + + // len = 1 => return value felt (i.e. legacy erc20 token) + // len > 1 => return value ByteArray (i.e. new erc20 token) + let name = if name.len() == 1 { + parse_cairo_short_string(&name[0]).unwrap() + } else { + ByteArray::cairo_deserialize(&name, 0) + .expect("Return value not ByteArray") + .to_string() + .expect("Return value not String") + }; + + let symbol = provider + .call( + FunctionCall { + contract_address, + entry_point_selector: get_selector_from_name("symbol").unwrap(), + calldata: vec![], + }, + BlockId::Tag(BlockTag::Pending), + ) + .await?; + let symbol = if symbol.len() == 1 { + parse_cairo_short_string(&symbol[0]).unwrap() + } else { + ByteArray::cairo_deserialize(&symbol, 0) + .expect("Return value not ByteArray") + .to_string() + .expect("Return value not String") + }; + + let decimals = provider + .call( + FunctionCall { + contract_address, + entry_point_selector: get_selector_from_name("decimals").unwrap(), + calldata: vec![], + }, + BlockId::Tag(BlockTag::Pending), + ) + .await?; + let decimals = u8::cairo_deserialize(&decimals, 0).expect("Return value not u8"); + + // Insert the token into the tokens table + self.query_queue.enqueue( + "INSERT INTO tokens (id, contract_address, name, symbol, decimals) VALUES (?, ?, ?, ?, ?)", + vec![ + Argument::String(format!("{:#x}", contract_address)), + Argument::FieldElement(contract_address), + Argument::String(name), + Argument::String(symbol), + Argument::Int(decimals.into()), + ], + ); + } + + // Now proceed with the transfer handling // Insert transfer event to erc20_transfers table { let insert_query = @@ -1200,7 +1295,7 @@ impl Sql { self.query_queue.enqueue( insert_query, vec![ - Argument::FieldElement(token_address), + Argument::FieldElement(contract_address), Argument::FieldElement(from), Argument::FieldElement(to), Argument::String(u256_to_sql_string(&amount)), @@ -1217,10 +1312,10 @@ impl Sql { // statements. // Fetch balances for both `from` and `to` addresses, update them and write back to db let query = sqlx::query_as::<_, (String, String)>( - "SELECT account_address, balance FROM erc20_balances WHERE token_address = ? AND account_address \ + "SELECT account_address, balance FROM balances WHERE contract_address = ? AND account_address \ IN (?, ?)", ) - .bind(format!("{:#x}", token_address)) + .bind(format!("{:#x}", contract_address)) .bind(format!("{:#x}", from)) .bind(format!("{:#x}", to)); @@ -1250,18 +1345,20 @@ impl Sql { let new_to_balance = if to != Felt::ZERO { to_balance.add(amount) } else { to_balance }; let update_query = " - INSERT INTO erc20_balances (account_address, token_address, balance) - VALUES (?, ?, ?) - ON CONFLICT (account_address, token_address) + INSERT INTO balances (id, balance, account_address, contract_address, token_id) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT (id) DO UPDATE SET balance = excluded.balance"; if from != Felt::ZERO { self.query_queue.enqueue( update_query, vec![ - Argument::FieldElement(from), - Argument::FieldElement(token_address), + Argument::String(format!("{:#x}:{:#x}", from, contract_address)), Argument::String(u256_to_sql_string(&new_from_balance)), + Argument::FieldElement(from), + Argument::FieldElement(contract_address), + Argument::String(format!("{:#x}", contract_address)), ], ); } @@ -1270,9 +1367,11 @@ impl Sql { self.query_queue.enqueue( update_query, vec![ - Argument::FieldElement(to), - Argument::FieldElement(token_address), + Argument::String(format!("{:#x}:{:#x}", to, contract_address)), Argument::String(u256_to_sql_string(&new_to_balance)), + Argument::FieldElement(to), + Argument::FieldElement(contract_address), + Argument::String(format!("{:#x}", contract_address)), ], ); } @@ -1282,13 +1381,79 @@ impl Sql { Ok(()) } - pub async fn handle_erc721_transfer( + pub async fn handle_erc721_transfer( &mut self, - token_address: Felt, + contract_address: Felt, from: Felt, to: Felt, token_id: U256, + provider: &P, ) -> Result<()> { + let balance_token_id = format!("{:#x}:{}", contract_address, u256_to_sql_string(&token_id)); + let token_exists: bool = + sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM tokens WHERE id = ?)") + .bind(balance_token_id.clone()) + .fetch_one(&self.pool) + .await?; + + if !token_exists { + // Fetch token information from the chain + let name = provider + .call( + FunctionCall { + contract_address, + entry_point_selector: get_selector_from_name("name").unwrap(), + calldata: vec![], + }, + BlockId::Tag(BlockTag::Pending), + ) + .await?; + + // len = 1 => return value felt (i.e. legacy erc721 token) + // len > 1 => return value ByteArray (i.e. new erc721 token) + let name = if name.len() == 1 { + parse_cairo_short_string(&name[0]).unwrap() + } else { + ByteArray::cairo_deserialize(&name, 0) + .expect("Return value not ByteArray") + .to_string() + .expect("Return value not String") + }; + + let symbol = provider + .call( + FunctionCall { + contract_address, + entry_point_selector: get_selector_from_name("symbol").unwrap(), + calldata: vec![], + }, + BlockId::Tag(BlockTag::Pending), + ) + .await?; + let symbol = if symbol.len() == 1 { + parse_cairo_short_string(&symbol[0]).unwrap() + } else { + ByteArray::cairo_deserialize(&symbol, 0) + .expect("Return value not ByteArray") + .to_string() + .expect("Return value not String") + }; + + let decimals = 0; + + // Insert the token into the tokens table + self.query_queue.enqueue( + "INSERT INTO tokens (id, contract_address, name, symbol, decimals) VALUES (?, ?, ?, ?, ?)", + vec![ + Argument::String(balance_token_id.clone()), + Argument::FieldElement(contract_address), + Argument::String(name), + Argument::String(symbol), + Argument::Int(decimals.into()), + ], + ); + } + // Insert transfer event to erc721_transfers table { let insert_query = @@ -1298,7 +1463,7 @@ impl Sql { self.query_queue.enqueue( insert_query, vec![ - Argument::FieldElement(token_address), + Argument::FieldElement(contract_address), Argument::FieldElement(from), Argument::FieldElement(to), Argument::String(u256_to_sql_string(&token_id)), @@ -1308,25 +1473,43 @@ impl Sql { // Update balances in erc721_balances table { + let update_query = " + INSERT INTO balances (id, balance, account_address, contract_address, token_id) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT (account_address, contract_address, token_id) + DO UPDATE SET balance = excluded.balance"; + let balance_token_id = + format!("{:#x}:{}", contract_address, u256_to_sql_string(&token_id)); + if from != Felt::ZERO { self.query_queue.enqueue( - "DELETE FROM erc721_balances WHERE account_address = ? AND token_address = ? AND token_id = ?", - vec![ - Argument::FieldElement(from), - Argument::FieldElement(token_address), - Argument::String(u256_to_sql_string(&token_id)), - ], + update_query, + vec![ + Argument::String(format!( + "{:#x}:{:#x}:{:#x}", + from, contract_address, token_id + )), + Argument::FieldElement(from), + Argument::FieldElement(contract_address), + Argument::String(u256_to_sql_string(&U256::from(1u8))), + Argument::String(balance_token_id.clone()), + ], ); } if to != Felt::ZERO { self.query_queue.enqueue( - "INSERT INTO erc721_balances (account_address, token_address, token_id) VALUES (?, ?, ?)", - vec![ - Argument::FieldElement(to), - Argument::FieldElement(token_address), - Argument::String(u256_to_sql_string(&token_id)), - ], + update_query, + vec![ + Argument::String(format!( + "{:#x}:{:#x}:{:#x}", + to, contract_address, token_id + )), + Argument::FieldElement(to), + Argument::FieldElement(contract_address), + Argument::String(u256_to_sql_string(&U256::from(0u8))), + Argument::String(balance_token_id.clone()), + ], ); } } diff --git a/crates/torii/core/src/types.rs b/crates/torii/core/src/types.rs index 0658ef98d8..d01941a485 100644 --- a/crates/torii/core/src/types.rs +++ b/crates/torii/core/src/types.rs @@ -122,3 +122,13 @@ impl FromStr for ErcType { } } } + +impl ToString for ErcType { + fn to_string(&self) -> String { + match self { + ErcType::ERC20 => "ERC20", + ErcType::ERC721 => "ERC721", + } + .to_string() + } +} diff --git a/crates/torii/migrations/20240803102207_add_erc.sql b/crates/torii/migrations/20240803102207_add_erc.sql index 173e1542fc..b36d7b5b28 100644 --- a/crates/torii/migrations/20240803102207_add_erc.sql +++ b/crates/torii/migrations/20240803102207_add_erc.sql @@ -1,15 +1,33 @@ -CREATE TABLE erc20_balances ( - account_address TEXT NOT NULL, - token_address TEXT NOT NULL, - balance TEXT NOT NULL, - PRIMARY KEY (account_address, token_address) +CREATE TABLE contracts ( + -- contract_address + id TEXT NOT NULL PRIMARY KEY, + contract_address TEXT NOT NULL, + contract_type TEXT NOT NULL, + head TEXT ); -CREATE TABLE erc721_balances ( +CREATE TABLE balances ( + -- account_address:contract_address:token_id + id TEXT NOT NULL PRIMARY KEY, + balance TEXT NOT NULL, account_address TEXT NOT NULL, - token_address TEXT NOT NULL, + contract_address TEXT NOT NULL, + -- contract_address:token_id token_id TEXT NOT NULL, - PRIMARY KEY (account_address, token_address, token_id) + FOREIGN KEY (token_id) REFERENCES tokens(id) +); + +CREATE INDEX balances_account_address ON balances (account_address); +CREATE INDEX balances_contract_address ON balances (contract_address); + +CREATE TABLE tokens ( + -- contract_address:token_id + id TEXT NOT NULL PRIMARY KEY, + contract_address TEXT NOT NULL, + name TEXT NOT NULL, + symbol TEXT NOT NULL, + decimals INTEGER NOT NULL + -- FOREIGN KEY (contract_address) REFERENCES contracts(id) ); CREATE TABLE erc20_transfers ( @@ -26,48 +44,4 @@ CREATE TABLE erc721_transfers ( from_address TEXT NOT NULL, to_address TEXT NOT NULL, token_id TEXT NOT NULL -); - --- these are metadata of the contracts which we would need to fetch from RPC separately --- not part of events engine - -CREATE TABLE erc20_contracts ( - token_address TEXT NOT NULL PRIMARY KEY, - name TEXT NOT NULL, - symbol TEXT NOT NULL, - decimals INTEGER NOT NULL, - total_supply TEXT NOT NULL -); - -CREATE TABLE erc721_contracts ( - token_address TEXT NOT NULL PRIMARY KEY, - name TEXT NOT NULL, - symbol TEXT NOT NULL, - total_supply TEXT NOT NULL -); - --- -- --- CREATE TABLE contracts ( --- id TEXT NOT NULL PRIMARY KEY, --- contract_address TEXT NOT NULL, --- contract_type TEXT NOT NULL, --- head TEXT NOT NULL, --- ) - --- CREATE TABLE balances ( --- id TEXT NOT NULL PRIMARY KEY, --- balance TEXT NOT NULL, --- account_address TEXT NOT NULL, --- contract_address TEXT NOT NULL, --- token_id TEXT, --- FOREIGN KEY (token_id) REFERENCES tokens(id), --- ) - --- CREATE INDEX balances_account_address ON balances (account_address); - --- CREATE TABLE tokens ( --- id TEXT NOT NULL PRIMARY KEY, --- uri TEXT NOT NULL, --- contract_address TEXT NOT NULL, --- FOREIGN KEY (contract_address) REFERENCES contracts(contract_address), --- ) \ No newline at end of file +); \ No newline at end of file