diff --git a/Cargo.lock b/Cargo.lock index 7167cde918..f41033d65a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2991,6 +2991,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.0" @@ -4332,6 +4347,19 @@ dependencies = [ "tokio-io-timeout", ] +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "iana-time-zone" version = "0.1.58" @@ -5234,6 +5262,24 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "ndarray" version = "0.13.1" @@ -5496,12 +5542,50 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "openssl" +version = "0.10.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" +dependencies = [ + "bitflags 2.4.1", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.38", +] + [[package]] name = "openssl-probe" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "openssl-sys" +version = "0.9.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "option-ext" version = "0.2.0" @@ -6333,10 +6417,12 @@ dependencies = [ "http-body", "hyper", "hyper-rustls 0.24.2", + "hyper-tls", "ipnet", "js-sys", "log", "mime", + "native-tls", "once_cell", "percent-encoding", "pin-project-lite", @@ -6347,6 +6433,7 @@ dependencies = [ "serde_urlencoded", "system-configuration", "tokio", + "tokio-native-tls", "tokio-rustls 0.24.1", "tower-service", "url", @@ -7985,6 +8072,16 @@ dependencies = [ "syn 2.0.38", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.23.4" @@ -8267,6 +8364,7 @@ version = "0.3.2" dependencies = [ "anyhow", "async-trait", + "base64 0.21.5", "camino", "chrono", "dojo-test-utils", @@ -8278,6 +8376,7 @@ dependencies = [ "lazy_static", "log", "once_cell", + "reqwest", "scarb", "scarb-ui", "serde", diff --git a/Cargo.toml b/Cargo.toml index 7eb6836b26..03682b4438 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,7 @@ lto = "fat" anyhow = "1.0.75" assert_matches = "1.5.0" async-trait = "0.1.68" +base64 = "0.21.2" blockifier = { git = "https://github.com/starkware-libs/blockifier" } cairo-lang-casm = "2.3.1" cairo-lang-compiler = "2.3.1" diff --git a/crates/dojo-world/src/metadata.rs b/crates/dojo-world/src/metadata.rs index 75559768ce..9726192215 100644 --- a/crates/dojo-world/src/metadata.rs +++ b/crates/dojo-world/src/metadata.rs @@ -67,6 +67,15 @@ impl<'de> Deserialize<'de> for Uri { } } +impl Uri { + pub fn cid(&self) -> Option<&str> { + match self { + Uri::Ipfs(value) => value.strip_prefix("ipfs://"), + _ => None, + } + } +} + #[derive(Default, Serialize, Deserialize, Debug, Clone)] pub struct WorldMetadata { pub name: Option, diff --git a/crates/torii/core/Cargo.toml b/crates/torii/core/Cargo.toml index d41296a556..9daf884848 100644 --- a/crates/torii/core/Cargo.toml +++ b/crates/torii/core/Cargo.toml @@ -20,6 +20,7 @@ hex.workspace = true lazy_static.workspace = true log = "0.4.17" once_cell.workspace = true +reqwest = { version = "0.11.22", features = [ "blocking" ]} scarb-ui.workspace = true serde.workspace = true serde_json.workspace = true @@ -32,6 +33,7 @@ tokio = { version = "1.32.0", features = [ "sync" ], default-features = true } tokio-stream = "0.1.11" tokio-util = "0.7.7" tracing.workspace = true +base64.workspace = true [dev-dependencies] camino.workspace = true diff --git a/crates/torii/core/src/processors/metadata_update.rs b/crates/torii/core/src/processors/metadata_update.rs index 7ed9c3005d..de8f25384e 100644 --- a/crates/torii/core/src/processors/metadata_update.rs +++ b/crates/torii/core/src/processors/metadata_update.rs @@ -1,14 +1,25 @@ -use anyhow::{Error, Ok, Result}; +use std::time::Duration; + +use anyhow::{Error, Result}; use async_trait::async_trait; +use base64::engine::general_purpose; +use base64::Engine as _; use dojo_world::contracts::world::WorldContractReader; +use dojo_world::metadata::{Uri, WorldMetadata}; +use reqwest::Client; use starknet::core::types::{BlockWithTxs, Event, InvokeTransactionReceipt}; use starknet::core::utils::parse_cairo_short_string; use starknet::providers::Provider; -use tracing::info; +use starknet_crypto::FieldElement; +use tokio_util::bytes::Bytes; +use tracing::{error, info}; use super::EventProcessor; use crate::sql::Sql; +const IPFS_URL: &str = "https://ipfs.io/ipfs/"; +const MAX_RETRY: u8 = 3; + #[derive(Default)] pub struct MetadataUpdateProcessor; @@ -45,7 +56,7 @@ where let resource = &event.data[0]; let uri_len: u8 = event.data[1].try_into().unwrap(); - let uri = if uri_len > 0 { + let uri_str = if uri_len > 0 { event.data[2..=uri_len as usize + 1] .iter() .map(parse_cairo_short_string) @@ -55,10 +66,74 @@ where "".to_string() }; - info!("Resource {:#x} metadata set: {}", resource, uri); + info!("Resource {:#x} metadata set: {}", resource, uri_str); + db.set_metadata(resource, &uri_str); - db.set_metadata(resource, uri); + let db = db.clone(); + let resource = *resource; + tokio::spawn(async move { + try_retrieve(db, resource, uri_str).await; + }); Ok(()) } } + +async fn try_retrieve(mut db: Sql, resource: FieldElement, uri_str: String) { + match metadata(uri_str.clone()).await { + Ok((metadata, icon_img, cover_img)) => { + db.update_metadata(&resource, &uri_str, &metadata, &icon_img, &cover_img) + .await + .unwrap(); + info!("Updated resource {resource:#x} metadata from ipfs"); + } + Err(e) => { + error!("Error retrieving resource {resource:#x} uri {uri_str}: {e}") + } + } +} + +async fn metadata(uri_str: String) -> Result<(WorldMetadata, Option, Option)> { + let uri = Uri::Ipfs(uri_str); + let cid = uri.cid().ok_or("Uri is malformed").map_err(Error::msg)?; + + let bytes = fetch_content(cid, MAX_RETRY).await?; + let metadata: WorldMetadata = serde_json::from_str(std::str::from_utf8(&bytes)?)?; + + let icon_img = fetch_image(&metadata.icon_uri).await; + let cover_img = fetch_image(&metadata.cover_uri).await; + + Ok((metadata, icon_img, cover_img)) +} + +async fn fetch_image(image_uri: &Option) -> Option { + if let Some(uri) = image_uri { + let data = fetch_content(uri.cid()?, MAX_RETRY).await.ok()?; + let encoded = general_purpose::STANDARD.encode(data); + return Some(encoded); + } + + None +} + +async fn fetch_content(cid: &str, mut retries: u8) -> Result { + while retries > 0 { + let response = Client::new().get(format!("{IPFS_URL}{}", cid)).send().await; + + match response { + Ok(response) => return response.bytes().await.map_err(|e| e.into()), + Err(e) => { + retries -= 1; + if retries > 0 { + info!("Fetch uri failure: {}", e); + tokio::time::sleep(Duration::from_secs(3)).await; + } + } + } + } + + Err(Error::msg(format!( + "Failed to pull data from IPFS after {} attempts, cid: {}", + MAX_RETRY, cid + ))) +} diff --git a/crates/torii/core/src/sql.rs b/crates/torii/core/src/sql.rs index 48dfba98a9..02d8028b39 100644 --- a/crates/torii/core/src/sql.rs +++ b/crates/torii/core/src/sql.rs @@ -5,6 +5,7 @@ use anyhow::{anyhow, Result}; use chrono::{DateTime, Utc}; use dojo_types::primitive::Primitive; use dojo_types::schema::Ty; +use dojo_world::metadata::WorldMetadata; use sqlx::pool::PoolConnection; use sqlx::{Executor, Pool, Sqlite}; use starknet::core::types::{Event, FieldElement, InvokeTransactionV1}; @@ -20,6 +21,7 @@ pub const FELT_DELIMITER: &str = "/"; #[path = "sql_test.rs"] mod test; +#[derive(Debug, Clone)] pub struct Sql { world_address: FieldElement, pool: Pool, @@ -174,7 +176,7 @@ impl Sql { self.query_queue.push(query); } - pub fn set_metadata(&mut self, resource: &FieldElement, uri: String) { + pub fn set_metadata(&mut self, resource: &FieldElement, uri: &str) { self.query_queue.push(format!( "INSERT INTO metadata (id, uri) VALUES ('{:#x}', '{}') ON CONFLICT(id) DO UPDATE SET \ id=excluded.id, updated_at=CURRENT_TIMESTAMP", @@ -182,6 +184,44 @@ impl Sql { )); } + pub async fn update_metadata( + &mut self, + resource: &FieldElement, + uri: &str, + metadata: &WorldMetadata, + icon_img: &Option, + cover_img: &Option, + ) -> Result<()> { + let mut image_columns = String::new(); + let mut image_values = String::new(); + let mut image_updated = String::new(); + + if let Some(icon) = icon_img { + image_columns = ", icon_img".to_string(); + image_values = format!(", '{}'", icon); + image_updated = ", icon_img=excluded.icon_img".to_string(); + } + + if let Some(cover) = cover_img { + image_columns = format!("{}, cover_img", image_columns); + image_values = format!("{}, '{}'", image_values, cover); + image_updated = format!("{}, cover_img=excluded.cover_img", image_updated); + } + + let json = serde_json::to_string(metadata).unwrap(); // safe unwrap + + let query = format!( + "INSERT INTO metadata (id, uri, json {image_columns}) VALUES ('{resource:#x}', \ + '{uri}', '{json}' {image_values}) ON CONFLICT(id) DO UPDATE SET id=excluded.id, \ + json=excluded.json, updated_at=CURRENT_TIMESTAMP {image_updated}" + ); + + self.query_queue.push(query); + self.execute().await?; + + Ok(()) + } + pub async fn entity(&self, model: String, key: FieldElement) -> Result> { let query = format!("SELECT * FROM {model} WHERE id = {key}"); let mut conn: PoolConnection = self.pool.acquire().await?; diff --git a/crates/torii/graphql/Cargo.toml b/crates/torii/graphql/Cargo.toml index a57cb9e30b..8371cdfe1f 100644 --- a/crates/torii/graphql/Cargo.toml +++ b/crates/torii/graphql/Cargo.toml @@ -14,7 +14,7 @@ async-graphql = { version = "6.0.7", features = [ "chrono", "dynamic-schema" ] } async-graphql-warp = "6.0.7" async-recursion = "1.0.5" async-trait.workspace = true -base64 = "0.21.2" +base64.workspace = true chrono.workspace = true dojo-types = { path = "../../dojo-types" } lazy_static.workspace = true diff --git a/crates/torii/graphql/src/mapping.rs b/crates/torii/graphql/src/mapping.rs index 7dd304c4e9..c56cdcff19 100644 --- a/crates/torii/graphql/src/mapping.rs +++ b/crates/torii/graphql/src/mapping.rs @@ -93,5 +93,16 @@ lazy_static! { pub static ref METADATA_TYPE_MAPPING: TypeMapping = IndexMap::from([ (Name::new("id"), TypeData::Simple(TypeRef::named(TypeRef::ID))), (Name::new("uri"), TypeData::Simple(TypeRef::named(TypeRef::STRING))), + (Name::new("json"), TypeData::Simple(TypeRef::named(TypeRef::STRING))), + (Name::new("icon_img"), TypeData::Simple(TypeRef::named(TypeRef::STRING))), + (Name::new("cover_img"), TypeData::Simple(TypeRef::named(TypeRef::STRING))), + ( + Name::new("created_at"), + TypeData::Simple(TypeRef::named(GraphqlType::DateTime.to_string())) + ), + ( + Name::new("updated_at"), + TypeData::Simple(TypeRef::named(GraphqlType::DateTime.to_string())) + ), ]); } diff --git a/crates/torii/graphql/src/object/metadata.rs b/crates/torii/graphql/src/object/metadata.rs index 0c3b465e43..eb2ee8890d 100644 --- a/crates/torii/graphql/src/object/metadata.rs +++ b/crates/torii/graphql/src/object/metadata.rs @@ -1,5 +1,3 @@ -use async_graphql::dynamic::Field; - use super::{ObjectTrait, TypeMapping}; use crate::mapping::METADATA_TYPE_MAPPING; use crate::query::constants::METADATA_TABLE; @@ -22,8 +20,4 @@ impl ObjectTrait for MetadataObject { fn table_name(&self) -> Option<&str> { Some(METADATA_TABLE) } - - fn related_fields(&self) -> Option> { - None - } } diff --git a/crates/torii/graphql/src/tests/types-test/Scarb.toml b/crates/torii/graphql/src/tests/types-test/Scarb.toml index 39f4e1c8a5..d1a063ce69 100644 --- a/crates/torii/graphql/src/tests/types-test/Scarb.toml +++ b/crates/torii/graphql/src/tests/types-test/Scarb.toml @@ -12,6 +12,14 @@ dojo = { path = "../../../../../dojo-core" } [[target.dojo]] build-external-contracts = [] +[tool.dojo.world] +name = "types-test" +description = "Graphql types testing" +# icon_uri = "file://assets/icon.png" +# cover_uri = "file://assets/cover.png" +# website = "https://dojoengine.org" +# socials.x = "https://twitter.com/dojostarknet" + [tool.dojo.env] rpc_url = "http://localhost:5050/" account_address = "0x517ececd29116499f4a1b64b094da79ba08dfd54a3edaa316134c41f8160973" diff --git a/crates/torii/graphql/src/tests/types-test/assets/cover.png b/crates/torii/graphql/src/tests/types-test/assets/cover.png new file mode 100644 index 0000000000..51717cfc53 Binary files /dev/null and b/crates/torii/graphql/src/tests/types-test/assets/cover.png differ diff --git a/crates/torii/graphql/src/tests/types-test/assets/icon.png b/crates/torii/graphql/src/tests/types-test/assets/icon.png new file mode 100644 index 0000000000..7f874c022f Binary files /dev/null and b/crates/torii/graphql/src/tests/types-test/assets/icon.png differ diff --git a/crates/torii/migrations/20231030212846_save_metadata.sql b/crates/torii/migrations/20231030212846_save_metadata.sql new file mode 100644 index 0000000000..cb28d372c6 --- /dev/null +++ b/crates/torii/migrations/20231030212846_save_metadata.sql @@ -0,0 +1,3 @@ +ALTER TABLE metadata ADD COLUMN json TEXT; +ALTER TABLE metadata ADD COLUMN icon_img TEXT; +ALTER TABLE metadata ADD COLUMN cover_img TEXT; \ No newline at end of file