From 8a81615730ef0b1f0d60e2bb8b9a754b68114ed1 Mon Sep 17 00:00:00 2001 From: Nasr Date: Tue, 17 Dec 2024 17:48:41 +0700 Subject: [PATCH 01/21] feat(torii-grpc): start rework to use 1 single query --- crates/torii/core/src/sql/cache.rs | 4 ++ crates/torii/grpc/src/server/mod.rs | 81 ++++++++++++----------------- 2 files changed, 37 insertions(+), 48 deletions(-) diff --git a/crates/torii/core/src/sql/cache.rs b/crates/torii/core/src/sql/cache.rs index 3d72bd093e..db7fcd5d55 100644 --- a/crates/torii/core/src/sql/cache.rs +++ b/crates/torii/core/src/sql/cache.rs @@ -41,6 +41,10 @@ impl ModelCache { } pub async fn models(&self, selectors: &[Felt]) -> Result, Error> { + if selectors.is_empty() { + return Ok(self.model_cache.read().await.values().cloned().collect()); + } + let mut schemas = Vec::with_capacity(selectors.len()); for selector in selectors { schemas.push(self.model(selector).await?); diff --git a/crates/torii/grpc/src/server/mod.rs b/crates/torii/grpc/src/server/mod.rs index 8a272cae8d..07084515e7 100644 --- a/crates/torii/grpc/src/server/mod.rs +++ b/crates/torii/grpc/src/server/mod.rs @@ -490,69 +490,54 @@ impl DojoWorld { return Ok((Vec::new(), 0)); } - // Query to get entity IDs and their model IDs - let mut query = if table == EVENT_MESSAGES_HISTORICAL_TABLE { - format!( - r#" - SELECT {table}.id, {table}.data, {table}.model_id, group_concat({model_relation_table}.model_id) as model_ids - FROM {table} - JOIN {model_relation_table} ON {table}.id = {model_relation_table}.entity_id - {where_clause} - GROUP BY {table}.event_id - ORDER BY {table}.event_id DESC - "# - ) - } else { - format!( - r#" - SELECT {table}.id, group_concat({model_relation_table}.model_id) as model_ids - FROM {table} - JOIN {model_relation_table} ON {table}.id = {model_relation_table}.entity_id - {where_clause} - GROUP BY {table}.id - ORDER BY {table}.event_id DESC - "# - ) - }; - - if limit.is_some() { - query += " LIMIT ?" - } - - if offset.is_some() { - query += " OFFSET ?" - } - if table == EVENT_MESSAGES_HISTORICAL_TABLE { let entities = - self.fetch_historical_event_messages(&query, None, limit, offset).await?; + self.fetch_historical_event_messages(&format!( + r#" + SELECT {table}.id, {table}.data, {table}.model_id, group_concat({model_relation_table}.model_id) as model_ids + FROM {table} + JOIN {model_relation_table} ON {table}.id = {model_relation_table}.entity_id + {where_clause} + GROUP BY {table}.event_id + ORDER BY {table}.event_id DESC + "# + ), None, limit, offset).await?; return Ok((entities, total_count)); } - let mut query = sqlx::query_as(&query); + // retrieve all schemas + let schemas = self + .model_cache + .models(&[]) + .await? + .iter() + .map(|m| m.schema.clone()) + .collect::>(); + let (query, count_query) = build_sql_query( + &schemas, + table, + entity_relation_column, + Some(&where_clause), + order_by, + limit, + offset, + )?; + let query = sqlx::query(&query); if let Some(hashed_keys) = hashed_keys { for key in hashed_keys.hashed_keys { let key = Felt::from_bytes_be_slice(&key); query = query.bind(format!("{:#x}", key)); } } - if let Some(entity_updated_after) = entity_updated_after.clone() { query = query.bind(entity_updated_after); } - query = query.bind(limit).bind(offset); - let db_entities: Vec<(String, String)> = query.fetch_all(&self.pool).await?; + let entities = query.fetch_all(&self.pool).await?; + let entities = db_entities + .iter() + .map(|row| map_row_to_entity(row, &schemas, dont_include_hashed_keys)) + .collect::, Error>>()?; - let entities = self - .fetch_entities( - table, - entity_relation_column, - db_entities, - dont_include_hashed_keys, - order_by, - entity_models, - ) - .await?; Ok((entities, total_count)) } From 060cf0ebb4617de935ce436e0c563f418477eae9 Mon Sep 17 00:00:00 2001 From: Nasr Date: Tue, 17 Dec 2024 21:12:04 +0700 Subject: [PATCH 02/21] hashed keys --- crates/torii/grpc/src/server/mod.rs | 49 ++++++++++++++--------------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/crates/torii/grpc/src/server/mod.rs b/crates/torii/grpc/src/server/mod.rs index 07084515e7..f9aaddf64a 100644 --- a/crates/torii/grpc/src/server/mod.rs +++ b/crates/torii/grpc/src/server/mod.rs @@ -464,33 +464,22 @@ impl DojoWorld { } }; - // count query that matches filter_ids - let count_query = format!( - r#" - SELECT count(*) - FROM {table} - {where_clause} - "# - ); - - // total count of rows without limit and offset - let mut count_query = sqlx::query_scalar(&count_query); - if let Some(hashed_keys) = &hashed_keys { - for key in &hashed_keys.hashed_keys { - let key = Felt::from_bytes_be_slice(key); - count_query = count_query.bind(format!("{:#x}", key)); + if table == EVENT_MESSAGES_HISTORICAL_TABLE { + let count_query = format!( + r#" + SELECT count(*) + FROM {table} + {where_clause} + "# + ); + let total_count = sqlx::query_scalar(&count_query) + .bind(entity_updated_after.clone()) + .fetch_one(&self.pool) + .await?; + if total_count == 0 { + return Ok((Vec::new(), 0)); } - } - - if let Some(entity_updated_after) = entity_updated_after.clone() { - count_query = count_query.bind(entity_updated_after); - } - let total_count = count_query.fetch_optional(&self.pool).await?.unwrap_or(0); - if total_count == 0 { - return Ok((Vec::new(), 0)); - } - if table == EVENT_MESSAGES_HISTORICAL_TABLE { let entities = self.fetch_historical_event_messages(&format!( r#" @@ -505,6 +494,14 @@ impl DojoWorld { return Ok((entities, total_count)); } + let count_query = format!( + r#" + SELECT count(*) + FROM {table} + {where_clause} + "# + ); + // retrieve all schemas let schemas = self .model_cache @@ -533,7 +530,7 @@ impl DojoWorld { query = query.bind(entity_updated_after); } let entities = query.fetch_all(&self.pool).await?; - let entities = db_entities + let entities = entities .iter() .map(|row| map_row_to_entity(row, &schemas, dont_include_hashed_keys)) .collect::, Error>>()?; From 3a45876a700553c11aa090b6f51bf5482f9a69c1 Mon Sep 17 00:00:00 2001 From: Nasr Date: Tue, 17 Dec 2024 21:53:08 +0700 Subject: [PATCH 03/21] start using build sql query singular with left joins --- crates/torii/core/src/model.rs | 10 +-- crates/torii/grpc/src/server/mod.rs | 103 +++++++++++++++------------- 2 files changed, 55 insertions(+), 58 deletions(-) diff --git a/crates/torii/core/src/model.rs b/crates/torii/core/src/model.rs index 4abd5c923a..4b9fbcda9b 100644 --- a/crates/torii/core/src/model.rs +++ b/crates/torii/core/src/model.rs @@ -123,8 +123,6 @@ pub fn build_sql_query( entity_relation_column: &str, where_clause: Option<&str>, order_by: Option<&str>, - limit: Option, - offset: Option, ) -> Result<(String, String), Error> { fn collect_columns(table_prefix: &str, path: &str, ty: &Ty, selections: &mut Vec) { match ty { @@ -205,13 +203,7 @@ pub fn build_sql_query( query += &format!(" ORDER BY {}.event_id DESC", table_name); } - if let Some(limit) = limit { - query += &format!(" LIMIT {}", limit); - } - - if let Some(offset) = offset { - query += &format!(" OFFSET {}", offset); - } + query += " LIMIT ? OFFSET ?"; Ok((query, count_query)) } diff --git a/crates/torii/grpc/src/server/mod.rs b/crates/torii/grpc/src/server/mod.rs index f9aaddf64a..e002c439db 100644 --- a/crates/torii/grpc/src/server/mod.rs +++ b/crates/torii/grpc/src/server/mod.rs @@ -13,6 +13,7 @@ use std::str::FromStr; use std::sync::Arc; use std::time::Duration; +use crypto_bigint::rand_core::le; use dojo_types::naming::compute_selector_from_tag; use dojo_types::primitive::{Primitive, PrimitiveError}; use dojo_types::schema::Ty; @@ -388,20 +389,16 @@ impl DojoWorld { async fn fetch_historical_event_messages( &self, query: &str, - keys_pattern: Option<&str>, + bind_values: Vec, limit: Option, offset: Option, ) -> Result, Error> { - let db_entities: Vec<(String, String, String, String)> = if keys_pattern.is_some() { - sqlx::query_as(query) - .bind(keys_pattern.unwrap()) - .bind(limit) - .bind(offset) - .fetch_all(&self.pool) - .await? - } else { - sqlx::query_as(query).bind(limit).bind(offset).fetch_all(&self.pool).await? - }; + let mut query = sqlx::query_as(query); + for value in bind_values { + query = query.bind(value); + } + let db_entities: Vec<(String, String, String, String)> = + query.bind(limit).bind(offset).fetch_all(&self.pool).await?; let mut entities = HashMap::new(); for (id, data, model_id, _) in db_entities { @@ -463,23 +460,42 @@ impl DojoWorld { } } }; + let mut bind_values = vec![]; + if let Some(hashed_keys) = hashed_keys { + bind_values = hashed_keys + .hashed_keys + .iter() + .map(|key| format!("{:#x}", Felt::from_bytes_be_slice(key))) + .collect::>() + } + if let Some(entity_updated_after) = entity_updated_after.clone() { + bind_values.push(entity_updated_after); + } - if table == EVENT_MESSAGES_HISTORICAL_TABLE { - let count_query = format!( - r#" - SELECT count(*) - FROM {table} - {where_clause} - "# - ); - let total_count = sqlx::query_scalar(&count_query) - .bind(entity_updated_after.clone()) - .fetch_one(&self.pool) - .await?; - if total_count == 0 { - return Ok((Vec::new(), 0)); - } + if let Some(limit) = limit { + bind_values.push(limit.to_string()); + } + if let Some(offset) = offset { + bind_values.push(offset.to_string()); + } + let count_query = format!( + r#" + SELECT count(*) + FROM {table} + {where_clause} + "# + ); + let mut count_query = sqlx::query_scalar(&count_query); + for value in &bind_values { + count_query = count_query.bind(value); + } + let total_count = count_query.fetch_one(&self.pool).await?; + if total_count == 0 { + return Ok((Vec::new(), 0)); + } + + if table == EVENT_MESSAGES_HISTORICAL_TABLE { let entities = self.fetch_historical_event_messages(&format!( r#" @@ -490,18 +506,10 @@ impl DojoWorld { GROUP BY {table}.event_id ORDER BY {table}.event_id DESC "# - ), None, limit, offset).await?; + ), bind_values, limit, offset).await?; return Ok((entities, total_count)); } - let count_query = format!( - r#" - SELECT count(*) - FROM {table} - {where_clause} - "# - ); - // retrieve all schemas let schemas = self .model_cache @@ -510,24 +518,21 @@ impl DojoWorld { .iter() .map(|m| m.schema.clone()) .collect::>(); - let (query, count_query) = build_sql_query( + let (query, _) = build_sql_query( &schemas, table, entity_relation_column, - Some(&where_clause), + if where_clause.is_empty() { + None + } else { + Some(&where_clause) + }, order_by, - limit, - offset, )?; - let query = sqlx::query(&query); - if let Some(hashed_keys) = hashed_keys { - for key in hashed_keys.hashed_keys { - let key = Felt::from_bytes_be_slice(&key); - query = query.bind(format!("{:#x}", key)); - } - } - if let Some(entity_updated_after) = entity_updated_after.clone() { - query = query.bind(entity_updated_after); + println!("query: {}", query); + let mut query = sqlx::query(&query); + for value in &bind_values { + query = query.bind(value); } let entities = query.fetch_all(&self.pool).await?; let entities = entities @@ -684,7 +689,7 @@ impl DojoWorld { if table == EVENT_MESSAGES_HISTORICAL_TABLE { let entities = self - .fetch_historical_event_messages(&models_query, Some(&keys_pattern), limit, offset) + .fetch_historical_event_messages(&models_query, vec![keys_pattern], limit, offset) .await?; return Ok((entities, total_count)); } From 9a4d414a67bc7c71a373d4920c4fd062cd9f93cb Mon Sep 17 00:00:00 2001 From: Nasr Date: Wed, 18 Dec 2024 15:19:42 +0700 Subject: [PATCH 04/21] optimize all queries --- crates/torii/core/src/model.rs | 26 +- crates/torii/grpc/src/server/mod.rs | 508 ++++++++-------------------- 2 files changed, 157 insertions(+), 377 deletions(-) diff --git a/crates/torii/core/src/model.rs b/crates/torii/core/src/model.rs index 4b9fbcda9b..9412a99119 100644 --- a/crates/torii/core/src/model.rs +++ b/crates/torii/core/src/model.rs @@ -120,9 +120,13 @@ impl ModelReader for ModelSQLReader { pub fn build_sql_query( schemas: &Vec, table_name: &str, + model_relation_table: &str, entity_relation_column: &str, where_clause: Option<&str>, + having_clause: Option<&str>, order_by: Option<&str>, + limit: Option, + offset: Option, ) -> Result<(String, String), Error> { fn collect_columns(table_prefix: &str, path: &str, ty: &Ty, selections: &mut Vec) { match ty { @@ -170,6 +174,7 @@ pub fn build_sql_query( // Add base table columns selections.push(format!("{}.id", table_name)); selections.push(format!("{}.keys", table_name)); + selections.push(format!("group_concat({model_relation_table}.model_id) as model_ids")); // Process each model schema for model in schemas { @@ -183,6 +188,10 @@ pub fn build_sql_query( collect_columns(&model_table, "", model, &mut selections); } + joins.push(format!( + "JOIN {model_relation_table} ON {table_name}.id = {model_relation_table}.entity_id" + )); + let selections_clause = selections.join(", "); let joins_clause = joins.join(" "); @@ -196,6 +205,13 @@ pub fn build_sql_query( count_query += &format!(" WHERE {}", where_clause); } + query += &format!(" GROUP BY {table_name}.id"); + + if let Some(having_clause) = having_clause { + query += &format!(" HAVING {}", having_clause); + count_query += &format!(" HAVING {}", having_clause); + } + // Use custom order by if provided, otherwise default to event_id DESC if let Some(order_clause) = order_by { query += &format!(" ORDER BY {}", order_clause); @@ -203,7 +219,13 @@ pub fn build_sql_query( query += &format!(" ORDER BY {}.event_id DESC", table_name); } - query += " LIMIT ? OFFSET ?"; + if let Some(limit) = limit { + query += &format!(" LIMIT {}", limit); + } + + if let Some(offset) = offset { + query += &format!(" OFFSET {}", offset); + } Ok((query, count_query)) } @@ -482,11 +504,13 @@ mod tests { let query = build_sql_query( &vec![position, player_config], "entities", + "entity_model", "internal_entity_id", None, None, None, None, + None, ) .unwrap(); diff --git a/crates/torii/grpc/src/server/mod.rs b/crates/torii/grpc/src/server/mod.rs index e002c439db..265e6b786b 100644 --- a/crates/torii/grpc/src/server/mod.rs +++ b/crates/torii/grpc/src/server/mod.rs @@ -13,7 +13,6 @@ use std::str::FromStr; use std::sync::Arc; use std::time::Duration; -use crypto_bigint::rand_core::le; use dojo_types::naming::compute_selector_from_tag; use dojo_types::primitive::{Primitive, PrimitiveError}; use dojo_types::schema::Ty; @@ -262,130 +261,6 @@ impl DojoWorld { row_events.iter().map(map_row_to_event).collect() } - #[allow(clippy::too_many_arguments)] - async fn fetch_entities( - &self, - table: &str, - entity_relation_column: &str, - entities: Vec<(String, String)>, - dont_include_hashed_keys: bool, - order_by: Option<&str>, - entity_models: Vec, - ) -> Result, Error> { - let entity_models = - entity_models.iter().map(|tag| compute_selector_from_tag(tag)).collect::>(); - - tracing::debug!( - "Fetching entities from table {table} with {} entity/model pairs", - entities.len() - ); - let start = std::time::Instant::now(); - - // Group entities by their model combinations - let mut model_groups: HashMap> = HashMap::new(); - for (entity_id, models_str) in entities { - model_groups.entry(models_str).or_default().push(entity_id); - } - tracing::debug!("Grouped into {} distinct model combinations", model_groups.len()); - - let mut all_entities = Vec::new(); - - let mut tx = self.pool.begin().await?; - tracing::debug!("Started database transaction"); - - // Create a temporary table to store entity IDs due to them potentially exceeding - // SQLite's parameters limit which is 999 - let temp_table_start = std::time::Instant::now(); - sqlx::query( - "CREATE TEMPORARY TABLE temp_entity_ids (id TEXT PRIMARY KEY, model_group TEXT)", - ) - .execute(&mut *tx) - .await?; - tracing::debug!("Created temporary table in {:?}", temp_table_start.elapsed()); - - // Insert all entity IDs into the temporary table - let insert_start = std::time::Instant::now(); - for (model_ids, entity_ids) in &model_groups { - for chunk in entity_ids.chunks(999) { - let placeholders = chunk.iter().map(|_| "(?, ?)").collect::>().join(","); - let query = format!( - "INSERT INTO temp_entity_ids (id, model_group) VALUES {}", - placeholders - ); - let mut query = sqlx::query(&query); - for id in chunk { - query = query.bind(id).bind(model_ids); - } - query.execute(&mut *tx).await?; - } - } - tracing::debug!( - "Inserted all entity IDs into temporary table in {:?}", - insert_start.elapsed() - ); - - let query_start = std::time::Instant::now(); - for (models_str, entity_ids) in &model_groups { - tracing::debug!("Processing model group with {} entities", entity_ids.len()); - let model_ids = models_str - .split(',') - .filter_map(|id| { - let model_id = Felt::from_str(id).unwrap(); - if entity_models.is_empty() || entity_models.contains(&model_id) { - Some(model_id) - } else { - None - } - }) - .collect::>(); - let schemas = self - .model_cache - .models(&model_ids) - .await? - .into_iter() - .map(|m| m.schema) - .collect::>(); - if schemas.is_empty() { - continue; - } - - let (entity_query, _) = build_sql_query( - &schemas, - table, - entity_relation_column, - Some(&format!( - "[{table}].id IN (SELECT id FROM temp_entity_ids WHERE model_group = ?)" - )), - order_by, - None, - None, - )?; - - let query = sqlx::query(&entity_query).bind(models_str); - let rows = query.fetch_all(&mut *tx).await?; - - let schemas = Arc::new(schemas); - - let group_entities: Result, Error> = rows - .par_iter() - .map(|row| map_row_to_entity(row, &schemas, dont_include_hashed_keys)) - .collect(); - - all_entities.extend(group_entities?); - } - tracing::debug!("Processed all model groups in {:?}", query_start.elapsed()); - - sqlx::query("DROP TABLE temp_entity_ids").execute(&mut *tx).await?; - tracing::debug!("Dropped temporary table"); - - tx.commit().await?; - tracing::debug!("Committed transaction"); - - tracing::debug!("Total fetch_entities operation took {:?}", start.elapsed()); - - Ok(all_entities) - } - async fn fetch_historical_event_messages( &self, query: &str, @@ -511,9 +386,11 @@ impl DojoWorld { } // retrieve all schemas + let entity_models = + entity_models.iter().map(|model| compute_selector_from_tag(model)).collect::>(); let schemas = self .model_cache - .models(&[]) + .models(&entity_models) .await? .iter() .map(|m| m.schema.clone()) @@ -521,15 +398,15 @@ impl DojoWorld { let (query, _) = build_sql_query( &schemas, table, + model_relation_table, entity_relation_column, - if where_clause.is_empty() { - None - } else { - Some(&where_clause) - }, + if where_clause.is_empty() { None } else { Some(&where_clause) }, + None, order_by, + limit, + offset, )?; - println!("query: {}", query); + let mut query = sqlx::query(&query); for value in &bind_values { query = query.bind(value); @@ -543,7 +420,6 @@ impl DojoWorld { Ok((entities, total_count)) } - #[allow(clippy::too_many_arguments)] pub(crate) async fn query_by_keys( &self, table: &str, @@ -559,158 +435,95 @@ impl DojoWorld { ) -> Result<(Vec, u32), Error> { let keys_pattern = build_keys_pattern(keys_clause)?; - // total count of rows that matches keys_pattern without limit and offset - let count_query = format!( - r#" - SELECT count(*) - FROM {table} - {} - "#, - if !keys_clause.models.is_empty() { - // split the model names to namespace and model - let model_ids = keys_clause - .models - .iter() - .map(|model| { - model - .split_once('-') - .ok_or(QueryError::InvalidNamespacedModel(model.clone())) - }) - .collect::, _>>()?; - // get the model selector from namespace and model and format - let model_ids_str = model_ids - .iter() - .map(|(namespace, model)| { - format!("'{:#x}'", compute_selector_from_names(namespace, model)) - }) - .collect::>() - .join(","); - format!( - r#" - JOIN {model_relation_table} ON {table}.id = {model_relation_table}.entity_id - WHERE {model_relation_table}.model_id IN ({}) - AND {table}.keys REGEXP ? - {} - "#, - model_ids_str, - if entity_updated_after.is_some() { - format!("AND {table}.updated_at >= ?") - } else { - String::new() - } - ) + let where_clause = format!( + "WHERE {table}.keys REGEXP ? {}", + if entity_updated_after.is_some() { + format!("AND {table}.updated_at >= ?") } else { - format!( - r#" - WHERE {table}.keys REGEXP ? - {} - "#, - if entity_updated_after.is_some() { - format!("AND {table}.updated_at >= ?") - } else { - String::new() - } - ) + String::new() } ); - let total_count = sqlx::query_scalar(&count_query) - .bind(&keys_pattern) - .bind(entity_updated_after.clone()) - .fetch_optional(&self.pool) - .await? - .unwrap_or(0); - if total_count == 0 { - return Ok((Vec::new(), 0)); + let mut bind_values = vec![keys_pattern]; + if let Some(entity_updated_after) = entity_updated_after.clone() { + bind_values.push(entity_updated_after); } - let mut models_query = if table == EVENT_MESSAGES_HISTORICAL_TABLE { - format!( - r#" - SELECT {table}.id, {table}.data, {table}.model_id, group_concat({model_relation_table}.model_id) as model_ids - FROM {table} - JOIN {model_relation_table} ON {table}.id = {model_relation_table}.entity_id - WHERE {table}.keys REGEXP ? - {} - GROUP BY {table}.event_id - "#, - if entity_updated_after.is_some() { - format!("AND {table}.updated_at >= ?") - } else { - String::new() - } - ) - } else { - format!( - r#" - SELECT {table}.id, group_concat({model_relation_table}.model_id) as model_ids - FROM {table} - JOIN {model_relation_table} ON {table}.id = {model_relation_table}.entity_id - WHERE {table}.keys REGEXP ? - {} - GROUP BY {table}.id - "#, - if entity_updated_after.is_some() { - format!("AND {table}.updated_at >= ?") - } else { - String::new() - } - ) - }; - - if !keys_clause.models.is_empty() { - // filter by models - models_query += &format!( - "HAVING {}", - keys_clause - .models - .iter() - .map(|model| { - let (namespace, name) = model - .split_once('-') - .ok_or(QueryError::InvalidNamespacedModel(model.clone()))?; - let model_id = compute_selector_from_names(namespace, name); - Ok(format!("INSTR(model_ids, '{:#x}') > 0", model_id)) - }) - .collect::, Error>>()? - .join(" OR ") - .as_str() - ); + if let Some(limit) = limit { + bind_values.push(limit.to_string()); + } + if let Some(offset) = offset { + bind_values.push(offset.to_string()); } - models_query += &format!(" ORDER BY {table}.event_id DESC"); - - if limit.is_some() { - models_query += " LIMIT ?"; + let count_query = format!( + r#" + SELECT count(*) + FROM {table} + {where_clause} + "# + ); + let mut count_query = sqlx::query_scalar(&count_query); + for value in &bind_values { + count_query = count_query.bind(value); } - if offset.is_some() { - models_query += " OFFSET ?"; + let total_count = count_query.fetch_one(&self.pool).await?; + if total_count == 0 { + return Ok((Vec::new(), 0)); } if table == EVENT_MESSAGES_HISTORICAL_TABLE { - let entities = self - .fetch_historical_event_messages(&models_query, vec![keys_pattern], limit, offset) - .await?; + let entities = self.fetch_historical_event_messages( + &format!( + r#" + SELECT {table}.id, {table}.data, {table}.model_id, group_concat({model_relation_table}.model_id) as model_ids + FROM {table} + JOIN {model_relation_table} ON {table}.id = {model_relation_table}.entity_id + {where_clause} + GROUP BY {table}.event_id + ORDER BY {table}.event_id DESC + "# + ), + bind_values, + limit, + offset + ).await?; return Ok((entities, total_count)); } - let mut query = sqlx::query_as(&models_query).bind(&keys_pattern); - if let Some(entity_updated_after) = entity_updated_after.clone() { - query = query.bind(entity_updated_after); + // retrieve all schemas + let entity_models = + entity_models.iter().map(|model| compute_selector_from_tag(model)).collect::>(); + let schemas = self + .model_cache + .models(&entity_models) + .await? + .iter() + .map(|m| m.schema.clone()) + .collect::>(); + + let (query, _) = build_sql_query( + &schemas, + table, + model_relation_table, + entity_relation_column, + Some(&where_clause), + None, + order_by, + limit, + offset, + )?; + + let mut query = sqlx::query(&query); + for value in &bind_values { + query = query.bind(value); } - query = query.bind(limit).bind(offset); - let db_entities: Vec<(String, String)> = query.fetch_all(&self.pool).await?; + let entities = query.fetch_all(&self.pool).await?; + let entities = entities + .iter() + .map(|row| map_row_to_entity(row, &schemas, dont_include_hashed_keys)) + .collect::, Error>>()?; - let entities = self - .fetch_entities( - table, - entity_relation_column, - db_entities, - dont_include_hashed_keys, - order_by, - entity_models, - ) - .await?; Ok((entities, total_count)) } @@ -818,8 +631,10 @@ impl DojoWorld { let (entity_query, count_query) = build_sql_query( &schemas, table, + model_relation_table, entity_relation_column, Some(&where_clause), + None, order_by, limit, offset, @@ -860,76 +675,46 @@ impl DojoWorld { entity_models: Vec, entity_updated_after: Option, ) -> Result<(Vec, u32), Error> { - let (where_clause, having_clause, join_clause, bind_values) = + let (where_clause, having_clauses, bind_values) = build_composite_clause(table, model_relation_table, &composite, entity_updated_after)?; - let count_query = if !having_clause.is_empty() { - format!( - r#" - SELECT COUNT(*) FROM ( - SELECT DISTINCT [{table}].id - FROM [{table}] - JOIN {model_relation_table} ON [{table}].id = {model_relation_table}.entity_id - {join_clause} - {where_clause} - GROUP BY [{table}].id - {having_clause} - ) as filtered_count - "#, - ) - } else { - format!( - r#" - SELECT COUNT(DISTINCT [{table}].id) - FROM [{table}] - JOIN {model_relation_table} ON [{table}].id = {model_relation_table}.entity_id - {join_clause} - {where_clause} - "#, - ) - }; + let entity_models = + entity_models.iter().map(|model| compute_selector_from_tag(model)).collect::>(); + let schemas = + self.model_cache.models(&entity_models).await?.into_iter().map(|m| m.schema).collect(); + let (query, count_query) = build_sql_query( + &schemas, + table, + model_relation_table, + entity_relation_column, + if where_clause.is_empty() { None } else { Some(&where_clause) }, + if having_clauses.is_empty() { None } else { Some(&having_clauses) }, + order_by, + limit, + offset, + )?; - let mut count_query = sqlx::query_scalar::<_, u32>(&count_query); + println!("count_query: {}", count_query); + let mut count_query = sqlx::query_scalar(&count_query); for value in &bind_values { count_query = count_query.bind(value); } - let total_count = count_query.fetch_optional(&self.pool).await?.unwrap_or(0); + + let total_count = count_query.fetch_one(&self.pool).await?; if total_count == 0 { return Ok((Vec::new(), 0)); } - let query = format!( - r#" - SELECT DISTINCT [{table}].id, group_concat({model_relation_table}.model_id) as model_ids - FROM [{table}] - JOIN {model_relation_table} ON [{table}].id = {model_relation_table}.entity_id - {join_clause} - {where_clause} - GROUP BY [{table}].id - {having_clause} - ORDER BY [{table}].event_id DESC - LIMIT ? OFFSET ? - "#, - ); - - let mut db_query = sqlx::query_as(&query); + let mut query = sqlx::query(&query); for value in &bind_values { - db_query = db_query.bind(value); + query = query.bind(value); } - db_query = db_query.bind(limit).bind(offset); - - let db_entities: Vec<(String, String)> = db_query.fetch_all(&self.pool).await?; + let db_entities = query.fetch_all(&self.pool).await?; - let entities = self - .fetch_entities( - table, - entity_relation_column, - db_entities, - dont_include_hashed_keys, - order_by, - entity_models, - ) - .await?; + let entities = db_entities + .par_iter() + .map(|row| map_row_to_entity(row, &schemas, dont_include_hashed_keys)) + .collect::, Error>>()?; Ok((entities, total_count)) } @@ -1247,8 +1032,15 @@ fn map_row_to_entity( dont_include_hashed_keys: bool, ) -> Result { let hashed_keys = Felt::from_str(&row.get::("id")).map_err(ParseError::FromStr)?; + let model_ids = row + .get::("model_ids") + .split(',') + .map(|id| Felt::from_str(id).map_err(ParseError::FromStr)) + .collect::, _>>()?; + let models = schemas .iter() + .filter(|schema| model_ids.contains(&compute_selector_from_tag(&schema.name()))) .map(|schema| { let mut ty = schema.clone(); map_row_to_ty("", &schema.name(), &mut ty, row)?; @@ -1300,13 +1092,11 @@ fn build_composite_clause( model_relation_table: &str, composite: &proto::types::CompositeClause, entity_updated_after: Option, -) -> Result<(String, String, String, Vec), Error> { +) -> Result<(String, String, Vec), Error> { let is_or = composite.operator == LogicalOperator::Or as i32; let mut where_clauses = Vec::new(); - let mut join_clauses = Vec::new(); let mut having_clauses = Vec::new(); let mut bind_values = Vec::new(); - let mut seen_models = HashMap::new(); for clause in &composite.clauses { match clause.clause_type.as_ref().unwrap() { @@ -1334,10 +1124,7 @@ fn build_composite_clause( .ok_or(QueryError::InvalidNamespacedModel(model.clone()))?; let model_id = compute_selector_from_names(namespace, model_name); - having_clauses.push(format!( - "INSTR(group_concat({model_relation_table}.model_id), '{:#x}') > 0", - model_id - )); + having_clauses.push(format!("INSTR(model_ids, '{:#x}') > 0", model_id)); } } ClauseType::Member(member) => { @@ -1356,65 +1143,34 @@ fn build_composite_clause( bind_values.push(comparison_value); let model = member.model.clone(); - // Get or create unique alias for this model - let alias = seen_models.entry(model.clone()).or_insert_with(|| { - let (namespace, model_name) = model - .split_once('-') - .ok_or(QueryError::InvalidNamespacedModel(model.clone())) - .unwrap(); - let model_id = compute_selector_from_names(namespace, model_name); - - // Add model check to having clause - having_clauses.push(format!( - "INSTR(group_concat({model_relation_table}.model_id), '{:#x}') > 0", - model_id - )); - - // Add join clause - join_clauses.push(format!( - "LEFT JOIN [{model}] AS [{model}] ON [{table}].id = \ - [{model}].internal_entity_id" - )); - - model.clone() - }); // Use the column name directly since it's already flattened where_clauses - .push(format!("([{alias}].[{}] {comparison_operator} ?)", member.member)); + .push(format!("([{model}].[{}] {comparison_operator} ?)", member.member)); } ClauseType::Composite(nested) => { // Handle nested composite by recursively building the clause - let (nested_where, nested_having, nested_join, nested_values) = - build_composite_clause( - table, - model_relation_table, - nested, - entity_updated_after.clone(), - )?; + let (nested_where, nested_having, nested_values) = build_composite_clause( + table, + model_relation_table, + nested, + entity_updated_after.clone(), + )?; if !nested_where.is_empty() { - where_clauses.push(format!("({})", nested_where.trim_start_matches("WHERE "))); + where_clauses.push(nested_where); } if !nested_having.is_empty() { - having_clauses - .push(format!("({})", nested_having.trim_start_matches("HAVING "))); + having_clauses.push(nested_having); } - join_clauses.extend( - nested_join - .split_whitespace() - .filter(|&s| s.starts_with("LEFT")) - .map(String::from), - ); bind_values.extend(nested_values); } } } - let join_clause = join_clauses.join(" "); let where_clause = if !where_clauses.is_empty() { format!( - "WHERE {} {}", + "{} {}", where_clauses.join(if is_or { " OR " } else { " AND " }), if let Some(entity_updated_after) = entity_updated_after.clone() { bind_values.push(entity_updated_after); @@ -1425,18 +1181,18 @@ fn build_composite_clause( ) } else if let Some(entity_updated_after) = entity_updated_after.clone() { bind_values.push(entity_updated_after); - format!("WHERE {table}.updated_at >= ?") + format!("{table}.updated_at >= ?") } else { String::new() }; let having_clause = if !having_clauses.is_empty() { - format!("HAVING {}", having_clauses.join(if is_or { " OR " } else { " AND " })) + format!("{}", having_clauses.join(if is_or { " OR " } else { " AND " })) } else { String::new() }; - Ok((where_clause, having_clause, join_clause, bind_values)) + Ok((where_clause, having_clause, bind_values)) } type ServiceResult = Result, Status>; From ae5fc97b392c8a46fa5e4faa2c2a6857e28a3cb9 Mon Sep 17 00:00:00 2001 From: Nasr Date: Wed, 18 Dec 2024 15:24:37 +0700 Subject: [PATCH 05/21] fix: count --- crates/torii/core/src/model.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/torii/core/src/model.rs b/crates/torii/core/src/model.rs index 9412a99119..449208908f 100644 --- a/crates/torii/core/src/model.rs +++ b/crates/torii/core/src/model.rs @@ -197,8 +197,12 @@ pub fn build_sql_query( let mut query = format!("SELECT {} FROM [{}] {}", selections_clause, table_name, joins_clause); - let mut count_query = - format!("SELECT COUNT(DISTINCT {}.id) FROM [{}] {}", table_name, table_name, joins_clause); + let mut count_query = format!( + "SELECT COUNT(DISTINCT {}.id, group_concat({model_relation_table}.model_id)) FROM [{}] {}", + table_name, + table_name, + joins_clause + ); if let Some(where_clause) = where_clause { query += &format!(" WHERE {}", where_clause); From 754805e5193ecb2e776a0e5f76fdafbe4905ef51 Mon Sep 17 00:00:00 2001 From: Nasr Date: Wed, 18 Dec 2024 15:27:30 +0700 Subject: [PATCH 06/21] f --- crates/torii/core/src/model.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/torii/core/src/model.rs b/crates/torii/core/src/model.rs index 449208908f..c8c49c5fab 100644 --- a/crates/torii/core/src/model.rs +++ b/crates/torii/core/src/model.rs @@ -198,7 +198,7 @@ pub fn build_sql_query( let mut query = format!("SELECT {} FROM [{}] {}", selections_clause, table_name, joins_clause); let mut count_query = format!( - "SELECT COUNT(DISTINCT {}.id, group_concat({model_relation_table}.model_id)) FROM [{}] {}", + "SELECT COUNT(DISTINCT {}.id), group_concat({model_relation_table}.model_id) as model_ids FROM [{}] {}", table_name, table_name, joins_clause From 55ed21643b496a274ab054de348b6fb8f83ce018 Mon Sep 17 00:00:00 2001 From: Nasr Date: Wed, 18 Dec 2024 15:49:55 +0700 Subject: [PATCH 07/21] fix count to use having --- crates/torii/core/src/model.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/torii/core/src/model.rs b/crates/torii/core/src/model.rs index c8c49c5fab..a188a2d225 100644 --- a/crates/torii/core/src/model.rs +++ b/crates/torii/core/src/model.rs @@ -198,10 +198,8 @@ pub fn build_sql_query( let mut query = format!("SELECT {} FROM [{}] {}", selections_clause, table_name, joins_clause); let mut count_query = format!( - "SELECT COUNT(DISTINCT {}.id), group_concat({model_relation_table}.model_id) as model_ids FROM [{}] {}", - table_name, - table_name, - joins_clause + "SELECT COUNT(*) FROM (SELECT {}.id FROM [{}] {} GROUP BY {}.id", + table_name, table_name, joins_clause, table_name ); if let Some(where_clause) = where_clause { @@ -216,6 +214,9 @@ pub fn build_sql_query( count_query += &format!(" HAVING {}", having_clause); } + // Close the subquery + count_query += ") AS filtered_entities"; + // Use custom order by if provided, otherwise default to event_id DESC if let Some(order_clause) = order_by { query += &format!(" ORDER BY {}", order_clause); From b0d3683690e65aa61adb70019b4d4602d32c02bb Mon Sep 17 00:00:00 2001 From: Nasr Date: Wed, 18 Dec 2024 15:54:04 +0700 Subject: [PATCH 08/21] f --- crates/torii/core/src/model.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/torii/core/src/model.rs b/crates/torii/core/src/model.rs index a188a2d225..9c42adc687 100644 --- a/crates/torii/core/src/model.rs +++ b/crates/torii/core/src/model.rs @@ -197,9 +197,10 @@ pub fn build_sql_query( let mut query = format!("SELECT {} FROM [{}] {}", selections_clause, table_name, joins_clause); + // Include model_ids in the subquery and put WHERE before GROUP BY let mut count_query = format!( - "SELECT COUNT(*) FROM (SELECT {}.id FROM [{}] {} GROUP BY {}.id", - table_name, table_name, joins_clause, table_name + "SELECT COUNT(*) FROM (SELECT {}.id, group_concat({}.model_id) as model_ids FROM [{}] {}", + table_name, model_relation_table, table_name, joins_clause ); if let Some(where_clause) = where_clause { @@ -208,6 +209,7 @@ pub fn build_sql_query( } query += &format!(" GROUP BY {table_name}.id"); + count_query += &format!(" GROUP BY {table_name}.id"); if let Some(having_clause) = having_clause { query += &format!(" HAVING {}", having_clause); From ee9f43a9f4f92a33cf96f71eae4e8b6c249bf124 Mon Sep 17 00:00:00 2001 From: Nasr Date: Wed, 18 Dec 2024 18:40:44 +0700 Subject: [PATCH 09/21] print query --- crates/torii/grpc/src/server/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/torii/grpc/src/server/mod.rs b/crates/torii/grpc/src/server/mod.rs index 265e6b786b..801a03beda 100644 --- a/crates/torii/grpc/src/server/mod.rs +++ b/crates/torii/grpc/src/server/mod.rs @@ -694,7 +694,6 @@ impl DojoWorld { offset, )?; - println!("count_query: {}", count_query); let mut count_query = sqlx::query_scalar(&count_query); for value in &bind_values { count_query = count_query.bind(value); @@ -705,6 +704,7 @@ impl DojoWorld { return Ok((Vec::new(), 0)); } + println!("query: {}", query); let mut query = sqlx::query(&query); for value in &bind_values { query = query.bind(value); From 496364f7623b9db88569741823571ae78f6c6686 Mon Sep 17 00:00:00 2001 From: Nasr Date: Wed, 18 Dec 2024 19:04:39 +0700 Subject: [PATCH 10/21] fix keys clause --- crates/torii/grpc/src/server/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/torii/grpc/src/server/mod.rs b/crates/torii/grpc/src/server/mod.rs index 801a03beda..baf304a0f6 100644 --- a/crates/torii/grpc/src/server/mod.rs +++ b/crates/torii/grpc/src/server/mod.rs @@ -436,7 +436,7 @@ impl DojoWorld { let keys_pattern = build_keys_pattern(keys_clause)?; let where_clause = format!( - "WHERE {table}.keys REGEXP ? {}", + "{table}.keys REGEXP ? {}", if entity_updated_after.is_some() { format!("AND {table}.updated_at >= ?") } else { @@ -460,7 +460,7 @@ impl DojoWorld { r#" SELECT count(*) FROM {table} - {where_clause} + WHERE {where_clause} "# ); let mut count_query = sqlx::query_scalar(&count_query); @@ -479,7 +479,7 @@ impl DojoWorld { SELECT {table}.id, {table}.data, {table}.model_id, group_concat({model_relation_table}.model_id) as model_ids FROM {table} JOIN {model_relation_table} ON {table}.id = {model_relation_table}.entity_id - {where_clause} + WHERE {where_clause} GROUP BY {table}.event_id ORDER BY {table}.event_id DESC "# From 78bc49e2cdaa7d0f5bc23950397e1f7dccaf9ecb Mon Sep 17 00:00:00 2001 From: Nasr Date: Wed, 18 Dec 2024 19:38:53 +0700 Subject: [PATCH 11/21] having clause keys --- crates/torii/grpc/src/server/mod.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/torii/grpc/src/server/mod.rs b/crates/torii/grpc/src/server/mod.rs index baf304a0f6..96c551580e 100644 --- a/crates/torii/grpc/src/server/mod.rs +++ b/crates/torii/grpc/src/server/mod.rs @@ -502,13 +502,18 @@ impl DojoWorld { .map(|m| m.schema.clone()) .collect::>(); + let having_clause = entity_models + .iter() + .map(|model| format!("INSTR(model_ids, '{:#x}') > 0", model)) + .collect::>() + .join(" OR "); let (query, _) = build_sql_query( &schemas, table, model_relation_table, entity_relation_column, Some(&where_clause), - None, + if !having_clause.is_empty() { Some(&having_clause) } else { None }, order_by, limit, offset, From 343d1e012e407e72df4617e42295e58d9036142a Mon Sep 17 00:00:00 2001 From: Nasr Date: Thu, 19 Dec 2024 13:07:01 +0700 Subject: [PATCH 12/21] having to hashed keys --- crates/torii/grpc/src/server/mod.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/torii/grpc/src/server/mod.rs b/crates/torii/grpc/src/server/mod.rs index 96c551580e..cc65650a28 100644 --- a/crates/torii/grpc/src/server/mod.rs +++ b/crates/torii/grpc/src/server/mod.rs @@ -395,13 +395,20 @@ impl DojoWorld { .iter() .map(|m| m.schema.clone()) .collect::>(); + + let having_clause = entity_models + .iter() + .map(|model| format!("INSTR(model_ids, '{:#x}') > 0", model)) + .collect::>() + .join(" OR "); + let (query, _) = build_sql_query( &schemas, table, model_relation_table, entity_relation_column, if where_clause.is_empty() { None } else { Some(&where_clause) }, - None, + if !having_clause.is_empty() { Some(&having_clause) } else { None }, order_by, limit, offset, From 99049e01ca10ce98842dbbc2136079c20aeaa9ad Mon Sep 17 00:00:00 2001 From: Nasr Date: Thu, 19 Dec 2024 13:11:57 +0700 Subject: [PATCH 13/21] having clause for composite --- crates/torii/grpc/src/server/mod.rs | 44 ++++++++++++++--------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/crates/torii/grpc/src/server/mod.rs b/crates/torii/grpc/src/server/mod.rs index cc65650a28..a92e360224 100644 --- a/crates/torii/grpc/src/server/mod.rs +++ b/crates/torii/grpc/src/server/mod.rs @@ -687,20 +687,27 @@ impl DojoWorld { entity_models: Vec, entity_updated_after: Option, ) -> Result<(Vec, u32), Error> { - let (where_clause, having_clauses, bind_values) = + let (where_clause, bind_values) = build_composite_clause(table, model_relation_table, &composite, entity_updated_after)?; let entity_models = entity_models.iter().map(|model| compute_selector_from_tag(model)).collect::>(); let schemas = self.model_cache.models(&entity_models).await?.into_iter().map(|m| m.schema).collect(); + + let having_clause = entity_models + .iter() + .map(|model| format!("INSTR(model_ids, '{:#x}') > 0", model)) + .collect::>() + .join(" OR "); + let (query, count_query) = build_sql_query( &schemas, table, model_relation_table, entity_relation_column, if where_clause.is_empty() { None } else { Some(&where_clause) }, - if having_clauses.is_empty() { None } else { Some(&having_clauses) }, + if having_clause.is_empty() { None } else { Some(&having_clause) }, order_by, limit, offset, @@ -1104,10 +1111,9 @@ fn build_composite_clause( model_relation_table: &str, composite: &proto::types::CompositeClause, entity_updated_after: Option, -) -> Result<(String, String, Vec), Error> { +) -> Result<(String, Vec), Error> { let is_or = composite.operator == LogicalOperator::Or as i32; let mut where_clauses = Vec::new(); - let mut having_clauses = Vec::new(); let mut bind_values = Vec::new(); for clause in &composite.clauses { @@ -1130,14 +1136,17 @@ fn build_composite_clause( where_clauses.push(format!("({table}.keys REGEXP ?)")); // Add model checks for specified models - for model in &keys.models { - let (namespace, model_name) = model - .split_once('-') - .ok_or(QueryError::InvalidNamespacedModel(model.clone()))?; - let model_id = compute_selector_from_names(namespace, model_name); + + // NOTE: disabled since we are now using the top level entity models - having_clauses.push(format!("INSTR(model_ids, '{:#x}') > 0", model_id)); - } + // for model in &keys.models { + // let (namespace, model_name) = model + // .split_once('-') + // .ok_or(QueryError::InvalidNamespacedModel(model.clone()))?; + // let model_id = compute_selector_from_names(namespace, model_name); + + // having_clauses.push(format!("INSTR(model_ids, '{:#x}') > 0", model_id)); + // } } ClauseType::Member(member) => { let comparison_operator = ComparisonOperator::from_repr(member.operator as usize) @@ -1162,7 +1171,7 @@ fn build_composite_clause( } ClauseType::Composite(nested) => { // Handle nested composite by recursively building the clause - let (nested_where, nested_having, nested_values) = build_composite_clause( + let (nested_where, nested_values) = build_composite_clause( table, model_relation_table, nested, @@ -1172,9 +1181,6 @@ fn build_composite_clause( if !nested_where.is_empty() { where_clauses.push(nested_where); } - if !nested_having.is_empty() { - having_clauses.push(nested_having); - } bind_values.extend(nested_values); } } @@ -1198,13 +1204,7 @@ fn build_composite_clause( String::new() }; - let having_clause = if !having_clauses.is_empty() { - format!("{}", having_clauses.join(if is_or { " OR " } else { " AND " })) - } else { - String::new() - }; - - Ok((where_clause, having_clause, bind_values)) + Ok((where_clause, bind_values)) } type ServiceResult = Result, Status>; From 56e4e6dfc61d50e1941b42bed73ef357ba991971 Mon Sep 17 00:00:00 2001 From: Nasr Date: Thu, 19 Dec 2024 13:12:41 +0700 Subject: [PATCH 14/21] fmt --- crates/torii/grpc/src/server/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/torii/grpc/src/server/mod.rs b/crates/torii/grpc/src/server/mod.rs index a92e360224..80de1a03b6 100644 --- a/crates/torii/grpc/src/server/mod.rs +++ b/crates/torii/grpc/src/server/mod.rs @@ -1136,7 +1136,7 @@ fn build_composite_clause( where_clauses.push(format!("({table}.keys REGEXP ?)")); // Add model checks for specified models - + // NOTE: disabled since we are now using the top level entity models // for model in &keys.models { From 39002ac6ab161493eb5d2110e05627665c505159 Mon Sep 17 00:00:00 2001 From: Nasr Date: Thu, 19 Dec 2024 13:38:55 +0700 Subject: [PATCH 15/21] fmt --- crates/torii/grpc/src/server/mod.rs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/crates/torii/grpc/src/server/mod.rs b/crates/torii/grpc/src/server/mod.rs index 80de1a03b6..cffc265fb1 100644 --- a/crates/torii/grpc/src/server/mod.rs +++ b/crates/torii/grpc/src/server/mod.rs @@ -427,6 +427,7 @@ impl DojoWorld { Ok((entities, total_count)) } + #[allow(clippy::too_many_arguments)] pub(crate) async fn query_by_keys( &self, table: &str, @@ -688,7 +689,7 @@ impl DojoWorld { entity_updated_after: Option, ) -> Result<(Vec, u32), Error> { let (where_clause, bind_values) = - build_composite_clause(table, model_relation_table, &composite, entity_updated_after)?; + build_composite_clause(table, &composite, entity_updated_after)?; let entity_models = entity_models.iter().map(|model| compute_selector_from_tag(model)).collect::>(); @@ -1108,7 +1109,6 @@ fn build_keys_pattern(clause: &proto::types::KeysClause) -> Result, ) -> Result<(String, Vec), Error> { @@ -1171,12 +1171,8 @@ fn build_composite_clause( } ClauseType::Composite(nested) => { // Handle nested composite by recursively building the clause - let (nested_where, nested_values) = build_composite_clause( - table, - model_relation_table, - nested, - entity_updated_after.clone(), - )?; + let (nested_where, nested_values) = + build_composite_clause(table, nested, entity_updated_after.clone())?; if !nested_where.is_empty() { where_clauses.push(nested_where); From e05122f69ddd1a2ae7e23345685f242b2df7e550 Mon Sep 17 00:00:00 2001 From: Nasr Date: Thu, 19 Dec 2024 13:42:42 +0700 Subject: [PATCH 16/21] fix: count --- crates/torii/grpc/src/server/mod.rs | 132 +++++++++++----------------- 1 file changed, 51 insertions(+), 81 deletions(-) diff --git a/crates/torii/grpc/src/server/mod.rs b/crates/torii/grpc/src/server/mod.rs index cffc265fb1..23a412f363 100644 --- a/crates/torii/grpc/src/server/mod.rs +++ b/crates/torii/grpc/src/server/mod.rs @@ -347,45 +347,6 @@ impl DojoWorld { bind_values.push(entity_updated_after); } - if let Some(limit) = limit { - bind_values.push(limit.to_string()); - } - if let Some(offset) = offset { - bind_values.push(offset.to_string()); - } - - let count_query = format!( - r#" - SELECT count(*) - FROM {table} - {where_clause} - "# - ); - let mut count_query = sqlx::query_scalar(&count_query); - for value in &bind_values { - count_query = count_query.bind(value); - } - let total_count = count_query.fetch_one(&self.pool).await?; - if total_count == 0 { - return Ok((Vec::new(), 0)); - } - - if table == EVENT_MESSAGES_HISTORICAL_TABLE { - let entities = - self.fetch_historical_event_messages(&format!( - r#" - SELECT {table}.id, {table}.data, {table}.model_id, group_concat({model_relation_table}.model_id) as model_ids - FROM {table} - JOIN {model_relation_table} ON {table}.id = {model_relation_table}.entity_id - {where_clause} - GROUP BY {table}.event_id - ORDER BY {table}.event_id DESC - "# - ), bind_values, limit, offset).await?; - return Ok((entities, total_count)); - } - - // retrieve all schemas let entity_models = entity_models.iter().map(|model| compute_selector_from_tag(model)).collect::>(); let schemas = self @@ -402,7 +363,7 @@ impl DojoWorld { .collect::>() .join(" OR "); - let (query, _) = build_sql_query( + let (query, count_query) = build_sql_query( &schemas, table, model_relation_table, @@ -414,6 +375,30 @@ impl DojoWorld { offset, )?; + let mut count_query = sqlx::query_scalar(&count_query); + for value in &bind_values { + count_query = count_query.bind(value); + } + let total_count = count_query.fetch_one(&self.pool).await?; + if total_count == 0 { + return Ok((Vec::new(), 0)); + } + + if table == EVENT_MESSAGES_HISTORICAL_TABLE { + let entities = + self.fetch_historical_event_messages(&format!( + r#" + SELECT {table}.id, {table}.data, {table}.model_id, group_concat({model_relation_table}.model_id) as model_ids + FROM {table} + JOIN {model_relation_table} ON {table}.id = {model_relation_table}.entity_id + {where_clause} + GROUP BY {table}.event_id + ORDER BY {table}.event_id DESC + "# + ), bind_values, limit, offset).await?; + return Ok((entities, total_count)); + } + let mut query = sqlx::query(&query); for value in &bind_values { query = query.bind(value); @@ -457,20 +442,33 @@ impl DojoWorld { bind_values.push(entity_updated_after); } - if let Some(limit) = limit { - bind_values.push(limit.to_string()); - } - if let Some(offset) = offset { - bind_values.push(offset.to_string()); - } + let entity_models = + entity_models.iter().map(|model| compute_selector_from_tag(model)).collect::>(); + let schemas = self + .model_cache + .models(&entity_models) + .await? + .iter() + .map(|m| m.schema.clone()) + .collect::>(); + + let having_clause = entity_models + .iter() + .map(|model| format!("INSTR(model_ids, '{:#x}') > 0", model)) + .collect::>() + .join(" OR "); + let (query, count_query) = build_sql_query( + &schemas, + table, + model_relation_table, + entity_relation_column, + Some(&where_clause), + if !having_clause.is_empty() { Some(&having_clause) } else { None }, + order_by, + limit, + offset, + )?; - let count_query = format!( - r#" - SELECT count(*) - FROM {table} - WHERE {where_clause} - "# - ); let mut count_query = sqlx::query_scalar(&count_query); for value in &bind_values { count_query = count_query.bind(value); @@ -499,34 +497,6 @@ impl DojoWorld { return Ok((entities, total_count)); } - // retrieve all schemas - let entity_models = - entity_models.iter().map(|model| compute_selector_from_tag(model)).collect::>(); - let schemas = self - .model_cache - .models(&entity_models) - .await? - .iter() - .map(|m| m.schema.clone()) - .collect::>(); - - let having_clause = entity_models - .iter() - .map(|model| format!("INSTR(model_ids, '{:#x}') > 0", model)) - .collect::>() - .join(" OR "); - let (query, _) = build_sql_query( - &schemas, - table, - model_relation_table, - entity_relation_column, - Some(&where_clause), - if !having_clause.is_empty() { Some(&having_clause) } else { None }, - order_by, - limit, - offset, - )?; - let mut query = sqlx::query(&query); for value in &bind_values { query = query.bind(value); From 124bc8f2fc2bfa30f74fe1d1d83bbf511906f704 Mon Sep 17 00:00:00 2001 From: Nasr Date: Thu, 19 Dec 2024 14:31:16 +0700 Subject: [PATCH 17/21] value --- crates/torii/grpc/src/server/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/torii/grpc/src/server/mod.rs b/crates/torii/grpc/src/server/mod.rs index 6227b1731c..b0e7d4ac45 100644 --- a/crates/torii/grpc/src/server/mod.rs +++ b/crates/torii/grpc/src/server/mod.rs @@ -1182,7 +1182,7 @@ fn build_composite_clause( // Use the column name directly since it's already flattened where_clauses - .push(format!("([{model}].[{}] {comparison_operator} ?)", member.member)); + .push(format!("([{model}].[{}] {comparison_operator} {value})", member.member)); } ClauseType::Composite(nested) => { // Handle nested composite by recursively building the clause From d5c4e7421d441761717c29415163389136ba610e Mon Sep 17 00:00:00 2001 From: Nasr Date: Thu, 19 Dec 2024 17:26:48 +0700 Subject: [PATCH 18/21] indices --- crates/torii/core/src/sql/mod.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/crates/torii/core/src/sql/mod.rs b/crates/torii/core/src/sql/mod.rs index 16388c5fb2..ab7dc3b358 100644 --- a/crates/torii/core/src/sql/mod.rs +++ b/crates/torii/core/src/sql/mod.rs @@ -766,6 +766,15 @@ impl Sql { TEXT, " ); + indices.push(format!( + "CREATE INDEX IF NOT EXISTS [idx_{table_id}_internal_entity_id] ON [{table_id}] \ + ([internal_entity_id]);" + )); + indices.push(format!( + "CREATE INDEX IF NOT EXISTS [idx_{table_id}_internal_event_message_id] ON [{table_id}] \ + ([internal_event_message_id]);" + )); + // Recursively add columns for all nested type add_columns_recursive( &path, From 3a1136ce2a3cdc283fa375899e10187c03a2e9b8 Mon Sep 17 00:00:00 2001 From: glihm Date: Sat, 21 Dec 2024 10:07:10 -0600 Subject: [PATCH 19/21] tests: fix tests --- crates/torii/core/src/model.rs | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/crates/torii/core/src/model.rs b/crates/torii/core/src/model.rs index 9c42adc687..cff842c115 100644 --- a/crates/torii/core/src/model.rs +++ b/crates/torii/core/src/model.rs @@ -522,16 +522,7 @@ mod tests { .unwrap(); let expected_query = - "SELECT entities.id, entities.keys, [Test-Position].[player] as \ - \"Test-Position.player\", [Test-Position].[vec.x] as \"Test-Position.vec.x\", \ - [Test-Position].[vec.y] as \"Test-Position.vec.y\", \ - [Test-Position].[test_everything] as \"Test-Position.test_everything\", \ - [Test-PlayerConfig].[favorite_item] as \"Test-PlayerConfig.favorite_item\", \ - [Test-PlayerConfig].[favorite_item.Some] as \ - \"Test-PlayerConfig.favorite_item.Some\", [Test-PlayerConfig].[items] as \ - \"Test-PlayerConfig.items\" FROM [entities] LEFT JOIN [Test-Position] ON entities.id \ - = [Test-Position].internal_entity_id LEFT JOIN [Test-PlayerConfig] ON entities.id = \ - [Test-PlayerConfig].internal_entity_id ORDER BY entities.event_id DESC"; + "SELECT entities.id, entities.keys, group_concat(entity_model.model_id) as model_ids, [Test-Position].[player] as \"Test-Position.player\", [Test-Position].[vec.x] as \"Test-Position.vec.x\", [Test-Position].[vec.y] as \"Test-Position.vec.y\", [Test-Position].[test_everything] as \"Test-Position.test_everything\", [Test-PlayerConfig].[favorite_item] as \"Test-PlayerConfig.favorite_item\", [Test-PlayerConfig].[favorite_item.Some] as \"Test-PlayerConfig.favorite_item.Some\", [Test-PlayerConfig].[items] as \"Test-PlayerConfig.items\" FROM [entities] LEFT JOIN [Test-Position] ON entities.id = [Test-Position].internal_entity_id LEFT JOIN [Test-PlayerConfig] ON entities.id = [Test-PlayerConfig].internal_entity_id JOIN entity_model ON entities.id = entity_model.entity_id GROUP BY entities.id ORDER BY entities.event_id DESC"; assert_eq!(query.0, expected_query); } } From 759c5b827d390fbd7278005245b4fd91404c69c6 Mon Sep 17 00:00:00 2001 From: glihm Date: Sat, 21 Dec 2024 10:18:48 -0600 Subject: [PATCH 20/21] fix: fmt --- crates/torii/core/src/model.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/crates/torii/core/src/model.rs b/crates/torii/core/src/model.rs index cff842c115..bf7d70de0d 100644 --- a/crates/torii/core/src/model.rs +++ b/crates/torii/core/src/model.rs @@ -522,7 +522,17 @@ mod tests { .unwrap(); let expected_query = - "SELECT entities.id, entities.keys, group_concat(entity_model.model_id) as model_ids, [Test-Position].[player] as \"Test-Position.player\", [Test-Position].[vec.x] as \"Test-Position.vec.x\", [Test-Position].[vec.y] as \"Test-Position.vec.y\", [Test-Position].[test_everything] as \"Test-Position.test_everything\", [Test-PlayerConfig].[favorite_item] as \"Test-PlayerConfig.favorite_item\", [Test-PlayerConfig].[favorite_item.Some] as \"Test-PlayerConfig.favorite_item.Some\", [Test-PlayerConfig].[items] as \"Test-PlayerConfig.items\" FROM [entities] LEFT JOIN [Test-Position] ON entities.id = [Test-Position].internal_entity_id LEFT JOIN [Test-PlayerConfig] ON entities.id = [Test-PlayerConfig].internal_entity_id JOIN entity_model ON entities.id = entity_model.entity_id GROUP BY entities.id ORDER BY entities.event_id DESC"; + "SELECT entities.id, entities.keys, group_concat(entity_model.model_id) as model_ids, \ + [Test-Position].[player] as \"Test-Position.player\", [Test-Position].[vec.x] as \ + \"Test-Position.vec.x\", [Test-Position].[vec.y] as \"Test-Position.vec.y\", \ + [Test-Position].[test_everything] as \"Test-Position.test_everything\", \ + [Test-PlayerConfig].[favorite_item] as \"Test-PlayerConfig.favorite_item\", \ + [Test-PlayerConfig].[favorite_item.Some] as \ + \"Test-PlayerConfig.favorite_item.Some\", [Test-PlayerConfig].[items] as \ + \"Test-PlayerConfig.items\" FROM [entities] LEFT JOIN [Test-Position] ON entities.id \ + = [Test-Position].internal_entity_id LEFT JOIN [Test-PlayerConfig] ON entities.id = \ + [Test-PlayerConfig].internal_entity_id JOIN entity_model ON entities.id = \ + entity_model.entity_id GROUP BY entities.id ORDER BY entities.event_id DESC"; assert_eq!(query.0, expected_query); } } From feecbdbc7b66a699251c3a15d77283a17bc009ed Mon Sep 17 00:00:00 2001 From: glihm Date: Sat, 21 Dec 2024 10:32:48 -0600 Subject: [PATCH 21/21] tests: fix by passing the list of expected models --- crates/torii/grpc/src/server/tests/entities_test.rs | 2 +- examples/spawn-and-move/Scarb.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/torii/grpc/src/server/tests/entities_test.rs b/crates/torii/grpc/src/server/tests/entities_test.rs index 22a38fda91..9791c09455 100644 --- a/crates/torii/grpc/src/server/tests/entities_test.rs +++ b/crates/torii/grpc/src/server/tests/entities_test.rs @@ -142,7 +142,7 @@ async fn test_entities_queries(sequencer: &RunnerCtx) { None, false, None, - vec![], + vec!["ns-Moves".to_string(), "ns-Position".to_string()], None, ) .await diff --git a/examples/spawn-and-move/Scarb.lock b/examples/spawn-and-move/Scarb.lock index 68efea17cb..76b825ff2e 100644 --- a/examples/spawn-and-move/Scarb.lock +++ b/examples/spawn-and-move/Scarb.lock @@ -31,7 +31,7 @@ dependencies = [ [[package]] name = "dojo_examples" -version = "1.0.5" +version = "1.0.8" dependencies = [ "armory", "bestiary",