From edf69c89895464d0ed79e80ee1f38eda269627ae Mon Sep 17 00:00:00 2001 From: samuel_rufi Date: Wed, 14 Aug 2024 12:38:51 +0200 Subject: [PATCH 01/15] feat(iota-json-rpc-api): Add explorer endpoints back --- .../src/embedded_reconfig_observer.rs | 3 +- .../iota-core/src/checkpoints/causal_order.rs | 3 +- .../checkpoints/checkpoint_executor/mod.rs | 3 +- .../src/checkpoints/checkpoint_output.rs | 3 +- crates/iota-indexer/src/apis/extended_api.rs | 66 ++++++++++++++- crates/iota-json-rpc-api/src/extended.rs | 34 +++++++- .../iota-json-rpc-types/src/iota_extended.rs | 80 ++++++++++++++++++- 7 files changed, 185 insertions(+), 7 deletions(-) diff --git a/crates/iota-benchmark/src/embedded_reconfig_observer.rs b/crates/iota-benchmark/src/embedded_reconfig_observer.rs index b1ad903a11b..a84e45f56b4 100644 --- a/crates/iota-benchmark/src/embedded_reconfig_observer.rs +++ b/crates/iota-benchmark/src/embedded_reconfig_observer.rs @@ -55,7 +55,8 @@ impl EmbeddedReconfigObserver { if new_epoch <= cur_epoch { trace!( cur_epoch, - new_epoch, "Ignored Committee from a previous or current epoch", + new_epoch, + "Ignored Committee from a previous or current epoch", ); return Ok(auth_agg); } diff --git a/crates/iota-core/src/checkpoints/causal_order.rs b/crates/iota-core/src/checkpoints/causal_order.rs index b907ea16b17..0d7c6cf5270 100644 --- a/crates/iota-core/src/checkpoints/causal_order.rs +++ b/crates/iota-core/src/checkpoints/causal_order.rs @@ -181,7 +181,8 @@ impl RWLockDependencyBuilder { for dep in reads { trace!( "Assuming additional dependency when constructing checkpoint {:?} -> {:?}", - digest, *dep + digest, + *dep ); v.insert(*dep); } diff --git a/crates/iota-core/src/checkpoints/checkpoint_executor/mod.rs b/crates/iota-core/src/checkpoints/checkpoint_executor/mod.rs index feebf06ec4e..966d2ff3bb4 100644 --- a/crates/iota-core/src/checkpoints/checkpoint_executor/mod.rs +++ b/crates/iota-core/src/checkpoints/checkpoint_executor/mod.rs @@ -740,7 +740,8 @@ async fn handle_execution_effects( if checkpoint.sequence_number > highest_seq + 1 { trace!( "Checkpoint {} is still executing. Highest executed = {}", - checkpoint.sequence_number, highest_seq + checkpoint.sequence_number, + highest_seq ); continue; } diff --git a/crates/iota-core/src/checkpoints/checkpoint_output.rs b/crates/iota-core/src/checkpoints/checkpoint_output.rs index 8eb91cd22f3..e6712c54af0 100644 --- a/crates/iota-core/src/checkpoints/checkpoint_output.rs +++ b/crates/iota-core/src/checkpoints/checkpoint_output.rs @@ -124,7 +124,8 @@ impl CheckpointOutput for LogCheckpointOutput { ) -> IotaResult { trace!( "Including following transactions in checkpoint {}: {:?}", - summary.sequence_number, contents + summary.sequence_number, + contents ); info!( "Creating checkpoint {:?} at epoch {}, sequence {}, previous digest {:?}, transactions count {}, content digest {:?}, end_of_epoch_data {:?}", diff --git a/crates/iota-indexer/src/apis/extended_api.rs b/crates/iota-indexer/src/apis/extended_api.rs index ce1d8d267ac..c08190f8179 100644 --- a/crates/iota-indexer/src/apis/extended_api.rs +++ b/crates/iota-indexer/src/apis/extended_api.rs @@ -7,7 +7,8 @@ use iota_json_rpc_api::{ internal_error, validate_limit, ExtendedApiServer, QUERY_MAX_RESULT_LIMIT_CHECKPOINTS, }; use iota_json_rpc_types::{ - CheckpointedObjectID, EpochInfo, EpochPage, IotaObjectResponseQuery, Page, QueryObjectsPage, + AddressMetrics, CheckpointedObjectID, EpochInfo, EpochMetrics, EpochMetricsPage, EpochPage, + IotaObjectResponseQuery, MoveCallMetrics, NetworkMetrics, Page, QueryObjectsPage, }; use iota_open_rpc::Module; use iota_types::iota_serde::BigInt; @@ -56,6 +57,46 @@ impl ExtendedApiServer for ExtendedApi { }) } + async fn get_epoch_metrics( + &self, + cursor: Option>, + limit: Option, + descending_order: Option, + ) -> RpcResult { + let limit = + validate_limit(limit, QUERY_MAX_RESULT_LIMIT_CHECKPOINTS).map_err(internal_error)?; + let epochs = self + .inner + .spawn_blocking(move |this| { + this.get_epochs( + cursor.map(|x| *x), + limit + 1, + descending_order.unwrap_or(false), + ) + }) + .await?; + + let mut epoch_metrics = epochs + .into_iter() + .map(|e| EpochMetrics { + epoch: e.epoch, + epoch_total_transactions: e.epoch_total_transactions, + first_checkpoint_id: e.first_checkpoint_id, + epoch_start_timestamp: e.epoch_start_timestamp, + end_of_epoch_info: e.end_of_epoch_info, + }) + .collect::>(); + + let has_next_page = epoch_metrics.len() > limit; + epoch_metrics.truncate(limit); + let next_cursor = epoch_metrics.last().map(|e| e.epoch); + Ok(Page { + data: epoch_metrics, + next_cursor: next_cursor.map(|id| id.into()), + has_next_page, + }) + } + async fn get_current_epoch(&self) -> RpcResult { let stored_epoch = self .inner @@ -64,6 +105,29 @@ impl ExtendedApiServer for ExtendedApi { EpochInfo::try_from(stored_epoch).map_err(Into::into) } + async fn get_network_metrics(&self) -> RpcResult { + Err(jsonrpsee::types::error::ErrorCode::MethodNotFound.into()) + } + + async fn get_move_call_metrics(&self) -> RpcResult { + Err(jsonrpsee::types::error::ErrorCode::MethodNotFound.into()) + } + + async fn get_latest_address_metrics(&self) -> RpcResult { + Err(jsonrpsee::types::error::ErrorCode::MethodNotFound.into()) + } + + async fn get_checkpoint_address_metrics(&self, _checkpoint: u64) -> RpcResult { + Err(jsonrpsee::types::error::ErrorCode::MethodNotFound.into()) + } + + async fn get_all_epoch_address_metrics( + &self, + _descending_order: Option, + ) -> RpcResult> { + Err(jsonrpsee::types::error::ErrorCode::MethodNotFound.into()) + } + async fn query_objects( &self, _query: IotaObjectResponseQuery, diff --git a/crates/iota-json-rpc-api/src/extended.rs b/crates/iota-json-rpc-api/src/extended.rs index 9fea355dc63..221d6b7bb9a 100644 --- a/crates/iota-json-rpc-api/src/extended.rs +++ b/crates/iota-json-rpc-api/src/extended.rs @@ -3,7 +3,8 @@ // SPDX-License-Identifier: Apache-2.0 use iota_json_rpc_types::{ - CheckpointedObjectID, EpochInfo, EpochPage, IotaObjectResponseQuery, QueryObjectsPage, + AddressMetrics, CheckpointedObjectID, EpochInfo, EpochMetricsPage, EpochPage, + IotaObjectResponseQuery, MoveCallMetrics, NetworkMetrics, QueryObjectsPage, }; use iota_open_rpc_macros::open_rpc; use iota_types::iota_serde::BigInt; @@ -27,6 +28,18 @@ pub trait ExtendedApi { descending_order: Option, ) -> RpcResult; + /// Return a list of epoch metrics, which is a subset of epoch info + #[method(name = "getEpochMetrics")] + async fn get_epoch_metrics( + &self, + /// optional paging cursor + cursor: Option>, + /// maximum number of items per page + limit: Option, + /// flag to return results in descending order + descending_order: Option, + ) -> RpcResult; + /// Return current epoch info #[method(name = "getCurrentEpoch")] async fn get_current_epoch(&self) -> RpcResult; @@ -44,6 +57,25 @@ pub trait ExtendedApi { limit: Option, ) -> RpcResult; + /// Return Network metrics + #[method(name = "getNetworkMetrics")] + async fn get_network_metrics(&self) -> RpcResult; + + /// Return Network metrics + #[method(name = "getMoveCallMetrics")] + async fn get_move_call_metrics(&self) -> RpcResult; + + /// Address related metrics + #[method(name = "getLatestAddressMetrics")] + async fn get_latest_address_metrics(&self) -> RpcResult; + #[method(name = "getCheckpointAddressMetrics")] + async fn get_checkpoint_address_metrics(&self, checkpoint: u64) -> RpcResult; + #[method(name = "getAllEpochAddressMetrics")] + async fn get_all_epoch_address_metrics( + &self, + descending_order: Option, + ) -> RpcResult>; + #[method(name = "getTotalTransactions")] async fn get_total_transactions(&self) -> RpcResult>; } diff --git a/crates/iota-json-rpc-types/src/iota_extended.rs b/crates/iota-json-rpc-types/src/iota_extended.rs index 63035433ed0..db8b1e502c1 100644 --- a/crates/iota-json-rpc-types/src/iota_extended.rs +++ b/crates/iota-json-rpc-types/src/iota_extended.rs @@ -20,6 +20,7 @@ use serde_with::{serde_as, DisplayFromStr}; use crate::Page; pub type EpochPage = Page>; +pub type EpochMetricsPage = Page>; #[serde_as] #[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)] @@ -57,6 +58,26 @@ impl EpochInfo { } } +/// a light-weight version of `EpochInfo` for faster loading +#[serde_as] +#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct EpochMetrics { + #[schemars(with = "BigInt")] + #[serde_as(as = "BigInt")] + pub epoch: EpochId, + #[schemars(with = "BigInt")] + #[serde_as(as = "BigInt")] + pub epoch_total_transactions: u64, + #[schemars(with = "BigInt")] + #[serde_as(as = "BigInt")] + pub first_checkpoint_id: CheckpointSequenceNumber, + #[schemars(with = "BigInt")] + #[serde_as(as = "BigInt")] + pub epoch_start_timestamp: u64, + pub end_of_epoch_info: Option, +} + #[serde_as] #[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)] #[serde(rename_all = "camelCase")] @@ -101,7 +122,52 @@ pub struct EndOfEpochInfo { } #[serde_as] -#[derive(Serialize, Deserialize, Debug, JsonSchema)] +#[derive(Serialize, Deserialize, Debug, JsonSchema, Clone)] +#[serde(rename_all = "camelCase")] +pub struct NetworkMetrics { + /// Current TPS - Transaction Blocks per Second. + pub current_tps: f64, + /// Peak TPS in the past 30 days + pub tps_30_days: f64, + /// Total number of packages published in the network + #[schemars(with = "BigInt")] + #[serde_as(as = "BigInt")] + pub total_packages: u64, + /// Total number of addresses seen in the network + #[schemars(with = "BigInt")] + #[serde_as(as = "BigInt")] + pub total_addresses: u64, + /// Total number of live objects in the network + #[schemars(with = "BigInt")] + #[serde_as(as = "BigInt")] + pub total_objects: u64, + /// Current epoch number + #[schemars(with = "BigInt")] + #[serde_as(as = "BigInt")] + pub current_epoch: u64, + /// Current checkpoint number + #[schemars(with = "BigInt")] + #[serde_as(as = "BigInt")] + pub current_checkpoint: u64, +} + +#[serde_as] +#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct MoveCallMetrics { + #[schemars(with = "Vec<(MoveFunctionName, BigInt)>")] + #[serde_as(as = "Vec<(_, BigInt)>")] + pub rank_3_days: Vec<(MoveFunctionName, usize)>, + #[schemars(with = "Vec<(MoveFunctionName, BigInt)>")] + #[serde_as(as = "Vec<(_, BigInt)>")] + pub rank_7_days: Vec<(MoveFunctionName, usize)>, + #[schemars(with = "Vec<(MoveFunctionName, BigInt)>")] + #[serde_as(as = "Vec<(_, BigInt)>")] + pub rank_30_days: Vec<(MoveFunctionName, usize)>, +} + +#[serde_as] +#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct MoveFunctionName { pub package: ObjectID, @@ -112,3 +178,15 @@ pub struct MoveFunctionName { #[serde_as(as = "DisplayFromStr")] pub function: Identifier, } + +#[serde_as] +#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct AddressMetrics { + pub checkpoint: u64, + pub epoch: u64, + pub timestamp_ms: u64, + pub cumulative_addresses: u64, + pub cumulative_active_addresses: u64, + pub daily_active_addresses: u64, +} From bf0c5077d3010210d7b6370cd01cc1e9672f177d Mon Sep 17 00:00:00 2001 From: samuel_rufi Date: Wed, 14 Aug 2024 15:10:22 +0200 Subject: [PATCH 02/15] feat(iota-json-rpc-api): Add metric processors back --- .../src/handlers/address_metrics_processor.rs | 124 ++++ crates/iota-indexer/src/handlers/mod.rs | 4 + .../handlers/move_call_metrics_processor.rs | 113 +++ .../src/handlers/network_metrics_processor.rs | 129 ++++ .../src/handlers/processor_orchestrator.rs | 83 +++ crates/iota-indexer/src/indexer.rs | 15 + crates/iota-indexer/src/indexer_reader.rs | 114 ++++ crates/iota-indexer/src/lib.rs | 3 + crates/iota-indexer/src/main.rs | 7 +- crates/iota-indexer/src/metrics.rs | 19 + .../src/models/address_metrics.rs | 112 +++ crates/iota-indexer/src/models/mod.rs | 4 + .../src/models/move_call_metrics.rs | 124 ++++ .../src/models/network_metrics.rs | 66 ++ .../src/models/tx_count_metrics.rs | 31 + .../src/store/indexer_analytics_store.rs | 77 +++ crates/iota-indexer/src/store/mod.rs | 4 + .../src/store/pg_indexer_analytical_store.rs | 643 ++++++++++++++++++ 18 files changed, 1671 insertions(+), 1 deletion(-) create mode 100644 crates/iota-indexer/src/handlers/address_metrics_processor.rs create mode 100644 crates/iota-indexer/src/handlers/move_call_metrics_processor.rs create mode 100644 crates/iota-indexer/src/handlers/network_metrics_processor.rs create mode 100644 crates/iota-indexer/src/handlers/processor_orchestrator.rs create mode 100644 crates/iota-indexer/src/models/address_metrics.rs create mode 100644 crates/iota-indexer/src/models/move_call_metrics.rs create mode 100644 crates/iota-indexer/src/models/network_metrics.rs create mode 100644 crates/iota-indexer/src/models/tx_count_metrics.rs create mode 100644 crates/iota-indexer/src/store/indexer_analytics_store.rs create mode 100644 crates/iota-indexer/src/store/pg_indexer_analytical_store.rs diff --git a/crates/iota-indexer/src/handlers/address_metrics_processor.rs b/crates/iota-indexer/src/handlers/address_metrics_processor.rs new file mode 100644 index 00000000000..1f473de6565 --- /dev/null +++ b/crates/iota-indexer/src/handlers/address_metrics_processor.rs @@ -0,0 +1,124 @@ +// Copyright (c) Mysten Labs, Inc. +// Modifications Copyright (c) 2024 IOTA Stiftung +// SPDX-License-Identifier: Apache-2.0 + +use tap::tap::TapFallible; +use tracing::{error, info}; + +use crate::metrics::IndexerMetrics; +use crate::store::IndexerAnalyticalStore; +use crate::types::IndexerResult; + +const ADDRESS_PROCESSOR_BATCH_SIZE: usize = 80000; +const PARALLELISM: usize = 10; + +pub struct AddressMetricsProcessor { + pub store: S, + metrics: IndexerMetrics, + pub address_processor_batch_size: usize, + pub address_processor_parallelism: usize, +} + +impl AddressMetricsProcessor + where + S: IndexerAnalyticalStore + Clone + Sync + Send + 'static, +{ + pub fn new(store: S, metrics: IndexerMetrics) -> AddressMetricsProcessor { + let address_processor_batch_size = std::env::var("ADDRESS_PROCESSOR_BATCH_SIZE") + .map(|s| s.parse::().unwrap_or(ADDRESS_PROCESSOR_BATCH_SIZE)) + .unwrap_or(ADDRESS_PROCESSOR_BATCH_SIZE); + let address_processor_parallelism = std::env::var("ADDRESS_PROCESSOR_PARALLELISM") + .map(|s| s.parse::().unwrap_or(PARALLELISM)) + .unwrap_or(PARALLELISM); + Self { + store, + metrics, + address_processor_batch_size, + address_processor_parallelism, + } + } + + pub async fn start(&self) -> IndexerResult<()> { + info!("Indexer address metrics async processor started..."); + let latest_tx_seq = self + .store + .get_address_metrics_last_processed_tx_seq() + .await?; + let mut last_processed_tx_seq = latest_tx_seq.unwrap_or_default().seq; + loop { + let mut latest_tx = self.store.get_latest_stored_transaction().await?; + while if let Some(tx) = latest_tx { + tx.tx_sequence_number + < last_processed_tx_seq + self.address_processor_batch_size as i64 + } else { + true + } { + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + latest_tx = self.store.get_latest_stored_transaction().await?; + } + + let mut persist_tasks = vec![]; + let batch_size = self.address_processor_batch_size; + let step_size = batch_size / self.address_processor_parallelism; + for chunk_start_tx_seq in (last_processed_tx_seq + 1 + ..last_processed_tx_seq + batch_size as i64 + 1) + .step_by(step_size) + { + let active_address_store = self.store.clone(); + persist_tasks.push(tokio::task::spawn_blocking(move || { + active_address_store.persist_active_addresses_in_tx_range( + chunk_start_tx_seq, + chunk_start_tx_seq + step_size as i64, + ) + })); + } + for chunk_start_tx_seq in (last_processed_tx_seq + 1 + ..last_processed_tx_seq + batch_size as i64 + 1) + .step_by(step_size) + { + let address_store = self.store.clone(); + persist_tasks.push(tokio::task::spawn_blocking(move || { + address_store.persist_addresses_in_tx_range( + chunk_start_tx_seq, + chunk_start_tx_seq + step_size as i64, + ) + })); + } + futures::future::join_all(persist_tasks) + .await + .into_iter() + .collect::, _>>() + .tap_err(|e| { + error!("Error joining address persist tasks: {:?}", e); + })? + .into_iter() + .collect::, _>>() + .tap_err(|e| { + error!("Error persisting addresses or active addresses: {:?}", e); + })?; + last_processed_tx_seq += self.address_processor_batch_size as i64; + info!( + "Persisted addresses and active addresses for tx seq: {}", + last_processed_tx_seq, + ); + self.metrics + .latest_address_metrics_tx_seq + .set(last_processed_tx_seq); + + let mut last_processed_tx = self.store.get_tx(last_processed_tx_seq).await?; + while last_processed_tx.is_none() { + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + last_processed_tx = self.store.get_tx(last_processed_tx_seq).await?; + } + // unwrap is safe here b/c we just checked that it's not None + let last_processed_cp = last_processed_tx.unwrap().checkpoint_sequence_number; + self.store + .calculate_and_persist_address_metrics(last_processed_cp) + .await?; + info!( + "Persisted address metrics for checkpoint: {}", + last_processed_cp + ); + } + } +} \ No newline at end of file diff --git a/crates/iota-indexer/src/handlers/mod.rs b/crates/iota-indexer/src/handlers/mod.rs index 5839be69eee..b7acbddead7 100644 --- a/crates/iota-indexer/src/handlers/mod.rs +++ b/crates/iota-indexer/src/handlers/mod.rs @@ -16,6 +16,10 @@ pub mod checkpoint_handler; pub mod committer; pub mod objects_snapshot_processor; pub mod tx_processor; +pub mod address_metrics_processor; +pub mod move_call_metrics_processor; +pub mod network_metrics_processor; +pub mod processor_orchestrator; #[derive(Debug)] pub struct CheckpointDataToCommit { diff --git a/crates/iota-indexer/src/handlers/move_call_metrics_processor.rs b/crates/iota-indexer/src/handlers/move_call_metrics_processor.rs new file mode 100644 index 00000000000..a631507f180 --- /dev/null +++ b/crates/iota-indexer/src/handlers/move_call_metrics_processor.rs @@ -0,0 +1,113 @@ +// Copyright (c) Mysten Labs, Inc. +// Modifications Copyright (c) 2024 IOTA Stiftung +// SPDX-License-Identifier: Apache-2.0 + +use tap::tap::TapFallible; +use tracing::{error, info}; + +use crate::metrics::IndexerMetrics; +use crate::store::IndexerAnalyticalStore; +use crate::types::IndexerResult; + +const MOVE_CALL_PROCESSOR_BATCH_SIZE: usize = 80000; +const PARALLELISM: usize = 10; + +pub struct MoveCallMetricsProcessor { + pub store: S, + metrics: IndexerMetrics, + pub move_call_processor_batch_size: usize, + pub move_call_processor_parallelism: usize, +} + +impl MoveCallMetricsProcessor + where + S: IndexerAnalyticalStore + Clone + Sync + Send + 'static, +{ + pub fn new(store: S, metrics: IndexerMetrics) -> MoveCallMetricsProcessor { + let move_call_processor_batch_size = std::env::var("MOVE_CALL_PROCESSOR_BATCH_SIZE") + .map(|s| s.parse::().unwrap_or(MOVE_CALL_PROCESSOR_BATCH_SIZE)) + .unwrap_or(MOVE_CALL_PROCESSOR_BATCH_SIZE); + let move_call_processor_parallelism = std::env::var("MOVE_CALL_PROCESSOR_PARALLELISM") + .map(|s| s.parse::().unwrap_or(PARALLELISM)) + .unwrap_or(PARALLELISM); + Self { + store, + metrics, + move_call_processor_batch_size, + move_call_processor_parallelism, + } + } + + pub async fn start(&self) -> IndexerResult<()> { + info!("Indexer move call metrics async processor started..."); + let latest_move_call_tx_seq = self.store.get_latest_move_call_tx_seq().await?; + let mut last_processed_tx_seq = latest_move_call_tx_seq.unwrap_or_default().seq; + let latest_move_call_epoch = self.store.get_latest_move_call_metrics().await?; + let mut last_processed_epoch = latest_move_call_epoch.unwrap_or_default().epoch; + loop { + let mut latest_tx = self.store.get_latest_stored_transaction().await?; + while if let Some(tx) = latest_tx { + tx.tx_sequence_number + < last_processed_tx_seq + self.move_call_processor_batch_size as i64 + } else { + true + } { + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + latest_tx = self.store.get_latest_stored_transaction().await?; + } + + let batch_size = self.move_call_processor_batch_size; + let step_size = batch_size / self.move_call_processor_parallelism; + let mut persist_tasks = vec![]; + for chunk_start_tx_seq in (last_processed_tx_seq + 1 + ..last_processed_tx_seq + batch_size as i64 + 1) + .step_by(step_size) + { + let move_call_store = self.store.clone(); + persist_tasks.push(tokio::task::spawn_blocking(move || { + move_call_store.persist_move_calls_in_tx_range( + chunk_start_tx_seq, + chunk_start_tx_seq + step_size as i64, + ) + })); + } + futures::future::join_all(persist_tasks) + .await + .into_iter() + .collect::, _>>() + .tap_err(|e| { + error!("Error joining move call persist tasks: {:?}", e); + })? + .into_iter() + .collect::, _>>() + .tap_err(|e| { + error!("Error persisting move calls: {:?}", e); + })?; + last_processed_tx_seq += batch_size as i64; + info!("Persisted move_calls at tx seq: {}", last_processed_tx_seq); + self.metrics + .latest_move_call_metrics_tx_seq + .set(last_processed_tx_seq); + + let mut tx = self.store.get_tx(last_processed_tx_seq).await?; + while tx.is_none() { + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + tx = self.store.get_tx(last_processed_tx_seq).await?; + } + let cp_seq = tx.unwrap().checkpoint_sequence_number; + let mut cp = self.store.get_cp(cp_seq).await?; + while cp.is_none() { + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + cp = self.store.get_cp(cp_seq).await?; + } + let end_epoch = cp.unwrap().epoch; + for epoch in last_processed_epoch + 1..end_epoch { + self.store + .calculate_and_persist_move_call_metrics(epoch) + .await?; + info!("Persisted move_call_metrics for epoch: {}", epoch); + } + last_processed_epoch = end_epoch - 1; + } + } +} \ No newline at end of file diff --git a/crates/iota-indexer/src/handlers/network_metrics_processor.rs b/crates/iota-indexer/src/handlers/network_metrics_processor.rs new file mode 100644 index 00000000000..e046ef8c84c --- /dev/null +++ b/crates/iota-indexer/src/handlers/network_metrics_processor.rs @@ -0,0 +1,129 @@ +// Copyright (c) Mysten Labs, Inc. +// Modifications Copyright (c) 2024 IOTA Stiftung +// SPDX-License-Identifier: Apache-2.0 + +use tap::tap::TapFallible; +use tracing::{error, info}; + +use crate::errors::IndexerError; +use crate::metrics::IndexerMetrics; +use crate::store::IndexerAnalyticalStore; +use crate::types::IndexerResult; + +const NETWORK_METRICS_PROCESSOR_BATCH_SIZE: usize = 10; +const PARALLELISM: usize = 1; + +pub struct NetworkMetricsProcessor { + pub store: S, + metrics: IndexerMetrics, + pub network_processor_metrics_batch_size: usize, + pub network_processor_metrics_parallelism: usize, +} + +impl NetworkMetricsProcessor + where + S: IndexerAnalyticalStore + Clone + Sync + Send + 'static, +{ + pub fn new(store: S, metrics: IndexerMetrics) -> NetworkMetricsProcessor { + let network_processor_metrics_batch_size = + std::env::var("NETWORK_PROCESSOR_METRICS_BATCH_SIZE") + .map(|s| { + s.parse::() + .unwrap_or(NETWORK_METRICS_PROCESSOR_BATCH_SIZE) + }) + .unwrap_or(NETWORK_METRICS_PROCESSOR_BATCH_SIZE); + let network_processor_metrics_parallelism = + std::env::var("NETWORK_PROCESSOR_METRICS_PARALLELISM") + .map(|s| s.parse::().unwrap_or(PARALLELISM)) + .unwrap_or(PARALLELISM); + Self { + store, + metrics, + network_processor_metrics_batch_size, + network_processor_metrics_parallelism, + } + } + + pub async fn start(&self) -> IndexerResult<()> { + info!("Indexer network metrics async processor started..."); + let latest_tx_count_metrics = self + .store + .get_latest_tx_count_metrics() + .await + .unwrap_or_default(); + let latest_epoch_peak_tps = self + .store + .get_latest_epoch_peak_tps() + .await + .unwrap_or_default(); + let mut last_processed_cp_seq = latest_tx_count_metrics + .unwrap_or_default() + .checkpoint_sequence_number; + let mut last_processed_peak_tps_epoch = latest_epoch_peak_tps.unwrap_or_default().epoch; + loop { + let mut latest_stored_checkpoint = self.store.get_latest_stored_checkpoint().await?; + while if let Some(cp) = latest_stored_checkpoint { + cp.sequence_number + < last_processed_cp_seq + self.network_processor_metrics_batch_size as i64 + } else { + true + } { + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + latest_stored_checkpoint = self.store.get_latest_stored_checkpoint().await?; + } + + info!( + "Persisting tx count metrics for checkpoint sequence number {}", + last_processed_cp_seq + ); + let batch_size = self.network_processor_metrics_batch_size; + let step_size = batch_size / self.network_processor_metrics_parallelism; + let mut persist_tasks = vec![]; + for chunk_start_cp in (last_processed_cp_seq + 1 + ..last_processed_cp_seq + batch_size as i64 + 1) + .step_by(step_size) + { + let store = self.store.clone(); + persist_tasks.push(tokio::task::spawn_blocking(move || { + store + .persist_tx_count_metrics(chunk_start_cp, chunk_start_cp + step_size as i64) + })); + } + futures::future::join_all(persist_tasks) + .await + .into_iter() + .collect::, _>>() + .tap_err(|e| { + error!("Error joining network persist tasks: {:?}", e); + })? + .into_iter() + .collect::, _>>() + .tap_err(|e| { + error!("Error persisting tx count metrics: {:?}", e); + })?; + last_processed_cp_seq += batch_size as i64; + info!( + "Persisted tx count metrics for checkpoint sequence number {}", + last_processed_cp_seq + ); + self.metrics + .latest_network_metrics_cp_seq + .set(last_processed_cp_seq); + + let end_cp = self + .store + .get_checkpoints_in_range(last_processed_cp_seq, last_processed_cp_seq + 1) + .await? + .first() + .ok_or(IndexerError::PostgresReadError( + "Cannot read checkpoint from PG for epoch peak TPS".to_string(), + ))? + .clone(); + for epoch in last_processed_peak_tps_epoch + 1..end_cp.epoch { + self.store.persist_epoch_peak_tps(epoch).await?; + last_processed_peak_tps_epoch = epoch; + info!("Persisted epoch peak TPS for epoch {}", epoch); + } + } + } +} \ No newline at end of file diff --git a/crates/iota-indexer/src/handlers/processor_orchestrator.rs b/crates/iota-indexer/src/handlers/processor_orchestrator.rs new file mode 100644 index 00000000000..0c78d1c0698 --- /dev/null +++ b/crates/iota-indexer/src/handlers/processor_orchestrator.rs @@ -0,0 +1,83 @@ +// Copyright (c) Mysten Labs, Inc. +// Modifications Copyright (c) 2024 IOTA Stiftung +// SPDX-License-Identifier: Apache-2.0 + +use futures::future::try_join_all; +use tracing::{error, info}; + +use crate::metrics::IndexerMetrics; +use crate::store::IndexerAnalyticalStore; + +use super::address_metrics_processor::AddressMetricsProcessor; +use super::move_call_metrics_processor::MoveCallMetricsProcessor; +use super::network_metrics_processor::NetworkMetricsProcessor; + +pub struct ProcessorOrchestrator { + store: S, + metrics: IndexerMetrics, +} + +impl ProcessorOrchestrator + where + S: IndexerAnalyticalStore + Clone + Send + Sync + 'static, +{ + pub fn new(store: S, metrics: IndexerMetrics) -> Self { + Self { store, metrics } + } + + pub async fn run_forever(&mut self) { + info!("Processor orchestrator started..."); + let network_metrics_processor = + NetworkMetricsProcessor::new(self.store.clone(), self.metrics.clone()); + let network_metrics_handle = tokio::task::spawn(async move { + loop { + let network_metrics_res = network_metrics_processor.start().await; + if let Err(e) = network_metrics_res { + tokio::time::sleep(std::time::Duration::from_secs(5)).await; + error!( + "Indexer network metrics processor failed with error {:?}, retrying in 5s...", + e + ); + } + } + }); + + let addr_metrics_processor = + AddressMetricsProcessor::new(self.store.clone(), self.metrics.clone()); + let addr_metrics_handle = tokio::task::spawn(async move { + loop { + let addr_metrics_res = addr_metrics_processor.start().await; + if let Err(e) = addr_metrics_res { + tokio::time::sleep(std::time::Duration::from_secs(5)).await; + error!( + "Indexer address metrics processor failed with error {:?}, retrying in 5s...", + e + ); + } + } + }); + + let move_call_metrics_processor = + MoveCallMetricsProcessor::new(self.store.clone(), self.metrics.clone()); + let move_call_metrics_handle = tokio::task::spawn(async move { + loop { + let move_call_metrics_res = move_call_metrics_processor.start().await; + if let Err(e) = move_call_metrics_res { + tokio::time::sleep(std::time::Duration::from_secs(5)).await; + error!( + "Indexer move call metrics processor failed with error {:?}, retrying in 5s...", + e + ); + } + } + }); + + try_join_all(vec![ + network_metrics_handle, + addr_metrics_handle, + move_call_metrics_handle, + ]) + .await + .expect("Processor orchestrator should not run into errors."); + } +} \ No newline at end of file diff --git a/crates/iota-indexer/src/indexer.rs b/crates/iota-indexer/src/indexer.rs index 41ba04d54e5..3c6296aac64 100644 --- a/crates/iota-indexer/src/indexer.rs +++ b/crates/iota-indexer/src/indexer.rs @@ -22,6 +22,8 @@ use crate::{ store::IndexerStore, IndexerConfig, }; +use crate::handlers::processor_orchestrator::ProcessorOrchestrator; +use crate::store::PgIndexerAnalyticalStore; const DOWNLOAD_QUEUE_SIZE: usize = 200; @@ -115,4 +117,17 @@ impl Indexer { Ok(()) } + + pub async fn start_analytical_worker( + store: PgIndexerAnalyticalStore, + metrics: IndexerMetrics, + ) -> Result<(), IndexerError> { + info!( + "Sui Indexer Analytical Worker (version {:?}) started...", + env!("CARGO_PKG_VERSION") + ); + let mut processor_orchestrator = ProcessorOrchestrator::new(store, metrics); + processor_orchestrator.run_forever().await; + Ok(()) + } } diff --git a/crates/iota-indexer/src/indexer_reader.rs b/crates/iota-indexer/src/indexer_reader.rs index 974f980dd94..bd84ca74754 100644 --- a/crates/iota-indexer/src/indexer_reader.rs +++ b/crates/iota-indexer/src/indexer_reader.rs @@ -19,6 +19,7 @@ use iota_json_rpc_types::{ IotaCoinMetadata, IotaEvent, IotaObjectDataFilter, IotaTransactionBlockEffects, IotaTransactionBlockEffectsAPI, IotaTransactionBlockResponse, TransactionFilter, }; + use iota_types::{ balance::Supply, base_types::{IotaAddress, ObjectID, ObjectRef, SequenceNumber, VersionNumber}, @@ -1510,6 +1511,119 @@ impl IndexerReader { .collect::>>() } + pub fn get_latest_network_metrics(&self) -> IndexerResult { + let metrics = self.run_query(|conn| { + diesel::sql_query("SELECT * FROM network_metrics;") + .get_result::(conn) + })?; + Ok(metrics.into()) + } + + pub fn get_latest_move_call_metrics(&self) -> IndexerResult { + let latest_3d_move_call_metrics = self.run_query(|conn| { + move_call_metrics::table + .filter(move_call_metrics::dsl::day.eq(3)) + .order(move_call_metrics::dsl::id.desc()) + .limit(10) + .load::(conn) + })?; + let latest_7d_move_call_metrics = self.run_query(|conn| { + move_call_metrics::table + .filter(move_call_metrics::dsl::day.eq(7)) + .order(move_call_metrics::dsl::id.desc()) + .limit(10) + .load::(conn) + })?; + let latest_30d_move_call_metrics = self.run_query(|conn| { + move_call_metrics::table + .filter(move_call_metrics::dsl::day.eq(30)) + .order(move_call_metrics::dsl::id.desc()) + .limit(10) + .load::(conn) + })?; + + let latest_3_days: Vec<(MoveFunctionName, usize)> = latest_3d_move_call_metrics + .into_iter() + .map(|m| m.try_into()) + .collect::, _>>()?; + let latest_7_days: Vec<(MoveFunctionName, usize)> = latest_7d_move_call_metrics + .into_iter() + .map(|m| m.try_into()) + .collect::, _>>()?; + let latest_30_days: Vec<(MoveFunctionName, usize)> = latest_30d_move_call_metrics + .into_iter() + .map(|m| m.try_into()) + .collect::, _>>()?; + // sort by call count desc. + let rank_3_days = latest_3_days + .into_iter() + .sorted_by(|a, b| b.1.cmp(&a.1)) + .collect::>(); + let rank_7_days = latest_7_days + .into_iter() + .sorted_by(|a, b| b.1.cmp(&a.1)) + .collect::>(); + let rank_30_days = latest_30_days + .into_iter() + .sorted_by(|a, b| b.1.cmp(&a.1)) + .collect::>(); + Ok(MoveCallMetrics { + rank_3_days, + rank_7_days, + rank_30_days, + }) + } + + pub fn get_latest_address_metrics(&self) -> IndexerResult { + let stored_address_metrics = self.run_query(|conn| { + address_metrics::table + .order(address_metrics::dsl::checkpoint.desc()) + .first::(conn) + })?; + Ok(stored_address_metrics.into()) + } + + pub fn get_checkpoint_address_metrics( + &self, + checkpoint_seq: u64, + ) -> IndexerResult { + let stored_address_metrics = self.run_query(|conn| { + address_metrics::table + .filter(address_metrics::dsl::checkpoint.eq(checkpoint_seq as i64)) + .first::(conn) + })?; + Ok(stored_address_metrics.into()) + } + + pub fn get_all_epoch_address_metrics( + &self, + descending_order: Option, + ) -> IndexerResult> { + let is_descending = descending_order.unwrap_or_default(); + let epoch_address_metrics_query = format!( + "WITH ranked_rows AS ( + SELECT + checkpoint, epoch, timestamp_ms, cumulative_addresses, cumulative_active_addresses, daily_active_addresses, + row_number() OVER(PARTITION BY epoch ORDER BY checkpoint DESC) as row_num + FROM + address_metrics + ) + SELECT + checkpoint, epoch, timestamp_ms, cumulative_addresses, cumulative_active_addresses, daily_active_addresses + FROM ranked_rows + WHERE row_num = 1 ORDER BY epoch {}", + if is_descending { "DESC" } else { "ASC" }, + ); + let epoch_address_metrics = self.run_query(|conn| { + diesel::sql_query(epoch_address_metrics_query).load::(conn) + })?; + + Ok(epoch_address_metrics + .into_iter() + .map(|stored_address_metrics| stored_address_metrics.into()) + .collect()) + } + pub(crate) async fn get_display_fields( &self, original_object: &iota_types::object::Object, diff --git a/crates/iota-indexer/src/lib.rs b/crates/iota-indexer/src/lib.rs index 6431e147a2a..add2a3606ed 100644 --- a/crates/iota-indexer/src/lib.rs +++ b/crates/iota-indexer/src/lib.rs @@ -74,6 +74,8 @@ pub struct IndexerConfig { pub fullnode_sync_worker: bool, #[clap(long)] pub rpc_server_worker: bool, + #[clap(long)] + pub analytical_worker: bool, } impl IndexerConfig { @@ -136,6 +138,7 @@ impl Default for IndexerConfig { reset_db: false, fullnode_sync_worker: true, rpc_server_worker: true, + analytical_worker: false, } } } diff --git a/crates/iota-indexer/src/main.rs b/crates/iota-indexer/src/main.rs index bca3acdfe82..a5c1e895aba 100644 --- a/crates/iota-indexer/src/main.rs +++ b/crates/iota-indexer/src/main.rs @@ -8,7 +8,9 @@ use iota_indexer::{ errors::IndexerError, indexer::Indexer, metrics::{start_prometheus_server, IndexerMetrics}, - store::PgIndexerStore, + store::{ + PgIndexerAnalyticalStore, PgIndexerStore, + }, IndexerConfig, }; use tracing::{error, info}; @@ -92,6 +94,9 @@ async fn main() -> Result<(), IndexerError> { return Indexer::start_writer(&indexer_config, store, indexer_metrics).await; } else if indexer_config.rpc_server_worker { return Indexer::start_reader(&indexer_config, ®istry, db_url).await; + } else if indexer_config.analytical_worker { + let store = PgIndexerAnalyticalStore::new(blocking_cp); + return Indexer::start_analytical_worker(store, indexer_metrics.clone()).await; } Ok(()) } diff --git a/crates/iota-indexer/src/metrics.rs b/crates/iota-indexer/src/metrics.rs index 804bd8c9e7f..4385b97d5ce 100644 --- a/crates/iota-indexer/src/metrics.rs +++ b/crates/iota-indexer/src/metrics.rs @@ -91,6 +91,10 @@ pub struct IndexerMetrics { pub latest_tx_checkpoint_sequence_number: IntGauge, pub latest_indexer_object_checkpoint_sequence_number: IntGauge, pub latest_object_snapshot_sequence_number: IntGauge, + // analytical + pub latest_move_call_metrics_tx_seq: IntGauge, + pub latest_address_metrics_tx_seq: IntGauge, + pub latest_network_metrics_cp_seq: IntGauge, // checkpoint E2E latency is: // fullnode_download_latency + checkpoint_index_latency + db_commit_latency pub checkpoint_download_bytes_size: IntGauge, @@ -241,6 +245,21 @@ impl IndexerMetrics { "Latest object snapshot sequence number from the Indexer", registry, ).unwrap(), + latest_move_call_metrics_tx_seq: register_int_gauge_with_registry!( + "latest_move_call_metrics_tx_seq", + "Latest move call metrics tx seq", + registry, + ).unwrap(), + latest_address_metrics_tx_seq: register_int_gauge_with_registry!( + "latest_address_metrics_tx_seq", + "Latest address metrics tx seq", + registry, + ).unwrap(), + latest_network_metrics_cp_seq: register_int_gauge_with_registry!( + "latest_network_metrics_cp_seq", + "Latest network metrics cp seq", + registry, + ).unwrap(), checkpoint_download_bytes_size: register_int_gauge_with_registry!( "checkpoint_download_bytes_size", "Size of the downloaded checkpoint in bytes", diff --git a/crates/iota-indexer/src/models/address_metrics.rs b/crates/iota-indexer/src/models/address_metrics.rs new file mode 100644 index 00000000000..bc3dfbdf0d3 --- /dev/null +++ b/crates/iota-indexer/src/models/address_metrics.rs @@ -0,0 +1,112 @@ +// Copyright (c) Mysten Labs, Inc. +// Modifications Copyright (c) 2024 IOTA Stiftung +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::HashMap; + +use diesel::prelude::*; +use diesel::sql_types::BigInt; + +use iota_json_rpc_types::AddressMetrics; + +use crate::schema::{active_addresses, address_metrics, addresses}; + +#[derive(Clone, Debug, Queryable, Insertable)] +#[diesel(table_name = addresses)] +pub struct StoredAddress { + pub address: Vec, + pub first_appearance_tx: i64, + pub first_appearance_time: i64, + pub last_appearance_tx: i64, + pub last_appearance_time: i64, +} + +#[derive(Clone, Debug, Queryable, Insertable)] +#[diesel(table_name = active_addresses)] +pub struct StoredActiveAddress { + pub address: Vec, + pub first_appearance_tx: i64, + pub first_appearance_time: i64, + pub last_appearance_tx: i64, + pub last_appearance_time: i64, +} + +impl From for StoredActiveAddress { + fn from(address: StoredAddress) -> Self { + StoredActiveAddress { + address: address.address, + first_appearance_tx: address.first_appearance_tx, + first_appearance_time: address.first_appearance_time, + last_appearance_tx: address.last_appearance_tx, + last_appearance_time: address.last_appearance_time, + } + } +} + +#[derive(Clone, Debug, Default, Queryable, Insertable, QueryableByName)] +#[diesel(table_name = address_metrics)] +pub struct StoredAddressMetrics { + #[diesel(sql_type = BigInt)] + pub checkpoint: i64, + #[diesel(sql_type = BigInt)] + pub epoch: i64, + #[diesel(sql_type = BigInt)] + pub timestamp_ms: i64, + #[diesel(sql_type = BigInt)] + pub cumulative_addresses: i64, + #[diesel(sql_type = BigInt)] + pub cumulative_active_addresses: i64, + #[diesel(sql_type = BigInt)] + pub daily_active_addresses: i64, +} + +impl From for AddressMetrics { + fn from(metrics: StoredAddressMetrics) -> Self { + Self { + checkpoint: metrics.checkpoint as u64, + epoch: metrics.epoch as u64, + timestamp_ms: metrics.timestamp_ms as u64, + cumulative_addresses: metrics.cumulative_addresses as u64, + cumulative_active_addresses: metrics.cumulative_active_addresses as u64, + daily_active_addresses: metrics.daily_active_addresses as u64, + } + } +} + +#[derive(Clone, Debug)] +pub struct AddressInfoToCommit { + pub address: Vec, + pub tx_seq: i64, + pub timestamp_ms: i64, +} + +pub fn dedup_addresses(addrs_to_commit: Vec) -> Vec { + let mut compressed_addr_map: HashMap<_, StoredAddress> = HashMap::new(); + for addr_to_commit in addrs_to_commit { + let entry = compressed_addr_map + .entry(addr_to_commit.address.clone()) + .or_insert_with(|| StoredAddress { + address: addr_to_commit.address.clone(), + first_appearance_time: addr_to_commit.timestamp_ms, + first_appearance_tx: addr_to_commit.tx_seq, + last_appearance_time: addr_to_commit.timestamp_ms, + last_appearance_tx: addr_to_commit.tx_seq, + }); + + if addr_to_commit.timestamp_ms < entry.first_appearance_time { + entry.first_appearance_time = addr_to_commit.timestamp_ms; + entry.first_appearance_tx = addr_to_commit.tx_seq; + } + if addr_to_commit.timestamp_ms > entry.last_appearance_time { + entry.last_appearance_time = addr_to_commit.timestamp_ms; + entry.last_appearance_tx = addr_to_commit.tx_seq; + } + } + compressed_addr_map.values().cloned().collect() +} + +#[derive(Clone, Debug)] +pub struct TxTimestampInfo { + pub tx_seq: i64, + pub timestamp_ms: i64, +} \ No newline at end of file diff --git a/crates/iota-indexer/src/models/mod.rs b/crates/iota-indexer/src/models/mod.rs index cfa32ddb294..a0e337e95c3 100644 --- a/crates/iota-indexer/src/models/mod.rs +++ b/crates/iota-indexer/src/models/mod.rs @@ -2,11 +2,15 @@ // Modifications Copyright (c) 2024 IOTA Stiftung // SPDX-License-Identifier: Apache-2.0 +pub mod address_metrics; pub mod checkpoints; pub mod display; pub mod epoch; pub mod events; +pub mod move_call_metrics; +pub mod network_metrics; pub mod objects; pub mod packages; pub mod transactions; +pub mod tx_count_metrics; pub mod tx_indices; diff --git a/crates/iota-indexer/src/models/move_call_metrics.rs b/crates/iota-indexer/src/models/move_call_metrics.rs new file mode 100644 index 00000000000..e7652426d9a --- /dev/null +++ b/crates/iota-indexer/src/models/move_call_metrics.rs @@ -0,0 +1,124 @@ +// Copyright (c) Mysten Labs, Inc. +// Modifications Copyright (c) 2024 IOTA Stiftung +// SPDX-License-Identifier: Apache-2.0 + +use std::str::FromStr; + +use diesel::prelude::*; +use diesel::sql_types::{BigInt, Binary, Text}; +use diesel::QueryableByName; + +use move_core_types::identifier::Identifier; +use sui_json_rpc_types::MoveFunctionName; +use sui_types::base_types::ObjectID; + +use crate::errors::IndexerError; +use crate::schema::{move_call_metrics, move_calls}; + +#[derive(Clone, Debug, Queryable, Insertable)] +#[diesel(table_name = move_calls)] +pub struct StoredMoveCall { + pub transaction_sequence_number: i64, + pub checkpoint_sequence_number: i64, + pub epoch: i64, + pub move_package: Vec, + pub move_module: String, + pub move_function: String, +} + +#[derive(Clone, Debug, Insertable)] +#[diesel(table_name = move_call_metrics)] +pub struct StoredMoveCallMetrics { + pub id: Option, + pub epoch: i64, + pub day: i64, + pub move_package: String, + pub move_module: String, + pub move_function: String, + pub count: i64, +} + +impl Default for StoredMoveCallMetrics { + fn default() -> Self { + Self { + id: None, + epoch: -1, + day: -1, + move_package: "".to_string(), + move_module: "".to_string(), + move_function: "".to_string(), + count: -1, + } + } +} + +// for auto-incremented id, the committed id is None, so Option, +// but when querying, the returned type is i64, thus a separate type is needed. +#[derive(Clone, Debug, Queryable)] +#[diesel(table_name = move_call_metrics)] +pub struct QueriedMoveCallMetrics { + pub id: i64, + pub epoch: i64, + pub day: i64, + pub move_package: String, + pub move_module: String, + pub move_function: String, + pub count: i64, +} + +impl TryInto<(MoveFunctionName, usize)> for QueriedMoveCallMetrics { + type Error = IndexerError; + + fn try_into(self) -> Result<(MoveFunctionName, usize), Self::Error> { + let package = ObjectID::from_str(&self.move_package)?; + let module = Identifier::from_str(&self.move_module)?; + let function = Identifier::from_str(&self.move_function)?; + Ok(( + MoveFunctionName { + package, + module, + function, + }, + self.count as usize, + )) + } +} + +impl From for StoredMoveCallMetrics { + fn from(q: QueriedMoveCallMetrics) -> Self { + StoredMoveCallMetrics { + id: Some(q.id), + epoch: q.epoch, + day: q.day, + move_package: q.move_package, + move_module: q.move_module, + move_function: q.move_function, + count: q.count, + } + } +} + +#[derive(QueryableByName, Debug, Clone, Default)] +pub struct QueriedMoveMetrics { + #[diesel(sql_type = BigInt)] + pub epoch: i64, + #[diesel(sql_type = BigInt)] + pub day: i64, + #[diesel(sql_type = Binary)] + pub move_package: Vec, + #[diesel(sql_type = Text)] + pub move_module: String, + #[diesel(sql_type = Text)] + pub move_function: String, + #[diesel(sql_type = BigInt)] + pub count: i64, +} + +pub fn build_move_call_metric_query(epoch: i64, days: i64) -> String { + format!("SELECT {}::BIGINT AS epoch, {}::BIGINT AS day, move_package, move_module, move_function, COUNT(*)::BIGINT AS count + FROM move_calls + WHERE epoch >= {} + GROUP BY move_package, move_module, move_function + ORDER BY count DESC + LIMIT 10;", epoch, days, epoch - days) +} \ No newline at end of file diff --git a/crates/iota-indexer/src/models/network_metrics.rs b/crates/iota-indexer/src/models/network_metrics.rs new file mode 100644 index 00000000000..7fa2d1c5219 --- /dev/null +++ b/crates/iota-indexer/src/models/network_metrics.rs @@ -0,0 +1,66 @@ +// Copyright (c) Mysten Labs, Inc. +// Modifications Copyright (c) 2024 IOTA Stiftung +// SPDX-License-Identifier: Apache-2.0 + +use diesel::prelude::*; +use diesel::sql_types::{BigInt, Double, Float8}; + +use sui_json_rpc_types::NetworkMetrics; + +use crate::schema::epoch_peak_tps; + +#[derive(Clone, Debug, Queryable, Insertable)] +#[diesel(table_name = epoch_peak_tps)] +pub struct StoredEpochPeakTps { + pub epoch: i64, + pub peak_tps: f64, + pub peak_tps_30d: f64, +} + +impl Default for StoredEpochPeakTps { + fn default() -> Self { + Self { + epoch: -1, + peak_tps: 0.0, + peak_tps_30d: 0.0, + } + } +} + +#[derive(QueryableByName, Debug, Clone, Default)] +pub struct StoredNetworkMetrics { + #[diesel(sql_type = Double)] + pub current_tps: f64, + #[diesel(sql_type = Double)] + pub tps_30_days: f64, + #[diesel(sql_type = BigInt)] + pub total_packages: i64, + #[diesel(sql_type = BigInt)] + pub total_addresses: i64, + #[diesel(sql_type = BigInt)] + pub total_objects: i64, + #[diesel(sql_type = BigInt)] + pub current_epoch: i64, + #[diesel(sql_type = BigInt)] + pub current_checkpoint: i64, +} + +impl From for NetworkMetrics { + fn from(db: StoredNetworkMetrics) -> Self { + Self { + current_tps: db.current_tps, + tps_30_days: db.tps_30_days, + total_packages: db.total_packages as u64, + total_addresses: db.total_addresses as u64, + total_objects: db.total_objects as u64, + current_epoch: db.current_epoch as u64, + current_checkpoint: db.current_checkpoint as u64, + } + } +} + +#[derive(Debug, QueryableByName)] +pub struct Tps { + #[diesel(sql_type = Float8)] + pub peak_tps: f64, +} \ No newline at end of file diff --git a/crates/iota-indexer/src/models/tx_count_metrics.rs b/crates/iota-indexer/src/models/tx_count_metrics.rs new file mode 100644 index 00000000000..388eff9bb89 --- /dev/null +++ b/crates/iota-indexer/src/models/tx_count_metrics.rs @@ -0,0 +1,31 @@ +// Copyright (c) Mysten Labs, Inc. +// Modifications Copyright (c) 2024 IOTA Stiftung +// SPDX-License-Identifier: Apache-2.0 + +use diesel::prelude::*; + +use crate::schema::tx_count_metrics; + +#[derive(Clone, Debug, Queryable, Insertable)] +#[diesel(table_name = tx_count_metrics)] +pub struct StoredTxCountMetrics { + pub checkpoint_sequence_number: i64, + pub epoch: i64, + pub timestamp_ms: i64, + pub total_transaction_blocks: i64, + pub total_successful_transaction_blocks: i64, + pub total_successful_transactions: i64, +} + +impl Default for StoredTxCountMetrics { + fn default() -> Self { + Self { + checkpoint_sequence_number: -1, + epoch: -1, + timestamp_ms: -1, + total_transaction_blocks: -1, + total_successful_transaction_blocks: -1, + total_successful_transactions: -1, + } + } +} \ No newline at end of file diff --git a/crates/iota-indexer/src/store/indexer_analytics_store.rs b/crates/iota-indexer/src/store/indexer_analytics_store.rs new file mode 100644 index 00000000000..0e964a3b931 --- /dev/null +++ b/crates/iota-indexer/src/store/indexer_analytics_store.rs @@ -0,0 +1,77 @@ +// Copyright (c) Mysten Labs, Inc. +// Modifications Copyright (c) 2024 IOTA Stiftung +// SPDX-License-Identifier: Apache-2.0 + +use async_trait::async_trait; + +use crate::models::checkpoints::StoredCheckpoint; +use crate::models::move_call_metrics::StoredMoveCallMetrics; +use crate::models::network_metrics::StoredEpochPeakTps; +use crate::models::transactions::{ + StoredTransaction, StoredTransactionCheckpoint, StoredTransactionSuccessCommandCount, + StoredTransactionTimestamp, TxSeq, +}; +use crate::models::tx_count_metrics::StoredTxCountMetrics; +use crate::types::IndexerResult; + +#[async_trait] +pub trait IndexerAnalyticalStore { + async fn get_latest_stored_transaction(&self) -> IndexerResult>; + async fn get_latest_stored_checkpoint(&self) -> IndexerResult>; + async fn get_checkpoints_in_range( + &self, + start_checkpoint: i64, + end_checkpoint: i64, + ) -> IndexerResult>; + async fn get_tx_timestamps_in_checkpoint_range( + &self, + start_checkpoint: i64, + end_checkpoint: i64, + ) -> IndexerResult>; + async fn get_tx_checkpoints_in_checkpoint_range( + &self, + start_checkpoint: i64, + end_checkpoint: i64, + ) -> IndexerResult>; + async fn get_tx_success_cmd_counts_in_checkpoint_range( + &self, + start_checkpoint: i64, + end_checkpoint: i64, + ) -> IndexerResult>; + async fn get_tx(&self, tx_sequence_number: i64) -> IndexerResult>; + async fn get_cp(&self, sequence_number: i64) -> IndexerResult>; + + // for network metrics including TPS and counts of objects etc. + async fn get_latest_tx_count_metrics(&self) -> IndexerResult>; + async fn get_latest_epoch_peak_tps(&self) -> IndexerResult>; + fn persist_tx_count_metrics( + &self, + start_checkpoint: i64, + end_checkpoint: i64, + ) -> IndexerResult<()>; + async fn persist_epoch_peak_tps(&self, epoch: i64) -> IndexerResult<()>; + + // for address metrics + async fn get_address_metrics_last_processed_tx_seq(&self) -> IndexerResult>; + fn persist_addresses_in_tx_range( + &self, + start_tx_seq: i64, + end_tx_seq: i64, + ) -> IndexerResult<()>; + fn persist_active_addresses_in_tx_range( + &self, + start_tx_seq: i64, + end_tx_seq: i64, + ) -> IndexerResult<()>; + async fn calculate_and_persist_address_metrics(&self, checkpoint: i64) -> IndexerResult<()>; + + // for move call metrics + async fn get_latest_move_call_metrics(&self) -> IndexerResult>; + async fn get_latest_move_call_tx_seq(&self) -> IndexerResult>; + fn persist_move_calls_in_tx_range( + &self, + start_tx_seq: i64, + end_tx_seq: i64, + ) -> IndexerResult<()>; + async fn calculate_and_persist_move_call_metrics(&self, epoch: i64) -> IndexerResult<()>; +} \ No newline at end of file diff --git a/crates/iota-indexer/src/store/mod.rs b/crates/iota-indexer/src/store/mod.rs index f0417495fd2..cd233140c2f 100644 --- a/crates/iota-indexer/src/store/mod.rs +++ b/crates/iota-indexer/src/store/mod.rs @@ -2,11 +2,15 @@ // Modifications Copyright (c) 2024 IOTA Stiftung // SPDX-License-Identifier: Apache-2.0 +pub(crate) use indexer_analytical_store::*; pub(crate) use indexer_store::*; +pub use pg_indexer_analytical_store::PgIndexerAnalyticalStore; pub use pg_indexer_store::PgIndexerStore; +mod indexer_analytics_store; pub mod indexer_store; pub mod module_resolver; +mod pg_indexer_analytical_store; mod pg_indexer_store; mod pg_partition_manager; pub mod query; diff --git a/crates/iota-indexer/src/store/pg_indexer_analytical_store.rs b/crates/iota-indexer/src/store/pg_indexer_analytical_store.rs new file mode 100644 index 00000000000..e9fa3a4bc02 --- /dev/null +++ b/crates/iota-indexer/src/store/pg_indexer_analytical_store.rs @@ -0,0 +1,643 @@ +// Copyright (c) Mysten Labs, Inc. +// Modifications Copyright (c) 2024 IOTA Stiftung +// SPDX-License-Identifier: Apache-2.0 + +use std::time::Duration; +use tap::tap::TapFallible; +use tracing::{error, info}; + +use async_trait::async_trait; +use core::result::Result::Ok; +use diesel::dsl::count; +use diesel::{ExpressionMethods, OptionalExtension}; +use diesel::{QueryDsl, RunQueryDsl}; +use sui_types::base_types::ObjectID; + +use crate::db::PgConnectionPool; +use crate::errors::{Context, IndexerError}; +use crate::models::address_metrics::StoredAddressMetrics; +use crate::models::checkpoints::StoredCheckpoint; +use crate::models::move_call_metrics::{ + build_move_call_metric_query, QueriedMoveCallMetrics, QueriedMoveMetrics, StoredMoveCallMetrics, +}; +use crate::models::network_metrics::{StoredEpochPeakTps, Tps}; +use crate::models::transactions::{ + StoredTransaction, StoredTransactionCheckpoint, StoredTransactionSuccessCommandCount, + StoredTransactionTimestamp, TxSeq, +}; +use crate::models::tx_count_metrics::StoredTxCountMetrics; +use crate::schema::{ + active_addresses, address_metrics, addresses, checkpoints, epoch_peak_tps, move_call_metrics, + move_calls, transactions, tx_count_metrics, +}; +use crate::store::diesel_macro::{read_only_blocking, transactional_blocking_with_retry}; +use crate::types::IndexerResult; + +use super::IndexerAnalyticalStore; + +#[derive(Clone)] +pub struct PgIndexerAnalyticalStore { + blocking_cp: PgConnectionPool, +} + +impl PgIndexerAnalyticalStore { + pub fn new(blocking_cp: PgConnectionPool) -> Self { + Self { blocking_cp } + } +} + +#[async_trait] +impl IndexerAnalyticalStore for PgIndexerAnalyticalStore { + async fn get_latest_stored_checkpoint(&self) -> IndexerResult> { + let latest_cp = read_only_blocking!(&self.blocking_cp, |conn| { + checkpoints::dsl::checkpoints + .order(checkpoints::sequence_number.desc()) + .first::(conn) + .optional() + }) + .context("Failed reading latest checkpoint from PostgresDB")?; + Ok(latest_cp) + } + + async fn get_latest_stored_transaction(&self) -> IndexerResult> { + let latest_tx = read_only_blocking!(&self.blocking_cp, |conn| { + transactions::dsl::transactions + .order(transactions::tx_sequence_number.desc()) + .first::(conn) + .optional() + }) + .context("Failed reading latest transaction from PostgresDB")?; + Ok(latest_tx) + } + + async fn get_checkpoints_in_range( + &self, + start_checkpoint: i64, + end_checkpoint: i64, + ) -> IndexerResult> { + let cps = read_only_blocking!(&self.blocking_cp, |conn| { + checkpoints::dsl::checkpoints + .filter(checkpoints::sequence_number.ge(start_checkpoint)) + .filter(checkpoints::sequence_number.lt(end_checkpoint)) + .order(checkpoints::sequence_number.asc()) + .load::(conn) + }) + .context("Failed reading checkpoints from PostgresDB")?; + Ok(cps) + } + + async fn get_tx_timestamps_in_checkpoint_range( + &self, + start_checkpoint: i64, + end_checkpoint: i64, + ) -> IndexerResult> { + let tx_timestamps = read_only_blocking!(&self.blocking_cp, |conn| { + transactions::dsl::transactions + .filter(transactions::dsl::checkpoint_sequence_number.ge(start_checkpoint)) + .filter(transactions::dsl::checkpoint_sequence_number.lt(end_checkpoint)) + .order(transactions::dsl::tx_sequence_number.asc()) + .select(( + transactions::dsl::tx_sequence_number, + transactions::dsl::timestamp_ms, + )) + .load::(conn) + }) + .context("Failed reading transaction timestamps from PostgresDB")?; + Ok(tx_timestamps) + } + + async fn get_tx_checkpoints_in_checkpoint_range( + &self, + start_checkpoint: i64, + end_checkpoint: i64, + ) -> IndexerResult> { + let tx_checkpoints = read_only_blocking!(&self.blocking_cp, |conn| { + transactions::dsl::transactions + .filter(transactions::dsl::checkpoint_sequence_number.ge(start_checkpoint)) + .filter(transactions::dsl::checkpoint_sequence_number.lt(end_checkpoint)) + .order(transactions::dsl::tx_sequence_number.asc()) + .select(( + transactions::dsl::tx_sequence_number, + transactions::dsl::checkpoint_sequence_number, + )) + .load::(conn) + }) + .context("Failed reading transaction checkpoints from PostgresDB")?; + Ok(tx_checkpoints) + } + + async fn get_tx_success_cmd_counts_in_checkpoint_range( + &self, + start_checkpoint: i64, + end_checkpoint: i64, + ) -> IndexerResult> { + let tx_success_cmd_counts = read_only_blocking!(&self.blocking_cp, |conn| { + transactions::dsl::transactions + .filter(transactions::dsl::checkpoint_sequence_number.ge(start_checkpoint)) + .filter(transactions::dsl::checkpoint_sequence_number.lt(end_checkpoint)) + .order(transactions::dsl::tx_sequence_number.asc()) + .select(( + transactions::dsl::tx_sequence_number, + transactions::dsl::checkpoint_sequence_number, + transactions::dsl::success_command_count, + transactions::dsl::timestamp_ms, + )) + .load::(conn) + }) + .context("Failed reading transaction success command counts from PostgresDB")?; + Ok(tx_success_cmd_counts) + } + async fn get_tx(&self, tx_sequence_number: i64) -> IndexerResult> { + let tx = read_only_blocking!(&self.blocking_cp, |conn| { + transactions::dsl::transactions + .filter(transactions::dsl::tx_sequence_number.eq(tx_sequence_number)) + .first::(conn) + .optional() + }) + .context("Failed reading transaction from PostgresDB")?; + Ok(tx) + } + + async fn get_cp(&self, sequence_number: i64) -> IndexerResult> { + let cp = read_only_blocking!(&self.blocking_cp, |conn| { + checkpoints::dsl::checkpoints + .filter(checkpoints::dsl::sequence_number.eq(sequence_number)) + .first::(conn) + .optional() + }) + .context("Failed reading checkpoint from PostgresDB")?; + Ok(cp) + } + + async fn get_latest_tx_count_metrics(&self) -> IndexerResult> { + let latest_tx_count = read_only_blocking!(&self.blocking_cp, |conn| { + tx_count_metrics::dsl::tx_count_metrics + .order(tx_count_metrics::dsl::checkpoint_sequence_number.desc()) + .first::(conn) + .optional() + }) + .context("Failed reading latest tx count metrics from PostgresDB")?; + Ok(latest_tx_count) + } + + async fn get_latest_epoch_peak_tps(&self) -> IndexerResult> { + let latest_network_metrics = read_only_blocking!(&self.blocking_cp, |conn| { + epoch_peak_tps::dsl::epoch_peak_tps + .order(epoch_peak_tps::dsl::epoch.desc()) + .first::(conn) + .optional() + }) + .context("Failed reading latest epoch peak TPS from PostgresDB")?; + Ok(latest_network_metrics) + } + + fn persist_tx_count_metrics( + &self, + start_checkpoint: i64, + end_checkpoint: i64, + ) -> IndexerResult<()> { + let tx_count_query = construct_checkpoint_tx_count_query(start_checkpoint, end_checkpoint); + info!("Persisting tx count metrics for cp {}", start_checkpoint); + transactional_blocking_with_retry!( + &self.blocking_cp, + |conn| { + diesel::sql_query(tx_count_query.clone()).execute(conn)?; + Ok::<(), IndexerError>(()) + }, + Duration::from_secs(10) + ) + .context("Failed persisting tx count metrics to PostgresDB")?; + info!("Persisted tx count metrics for cp {}", start_checkpoint); + Ok(()) + } + + async fn persist_epoch_peak_tps(&self, epoch: i64) -> IndexerResult<()> { + let epoch_peak_tps_query = construct_peak_tps_query(epoch, 1); + let peak_tps_30d_query = construct_peak_tps_query(epoch, 30); + let epoch_tps: Tps = + read_only_blocking!(&self.blocking_cp, |conn| diesel::RunQueryDsl::get_result( + diesel::sql_query(epoch_peak_tps_query), + conn + )) + .context("Failed reading epoch peak TPS from PostgresDB")?; + let tps_30d: Tps = + read_only_blocking!(&self.blocking_cp, |conn| diesel::RunQueryDsl::get_result( + diesel::sql_query(peak_tps_30d_query), + conn + )) + .context("Failed reading 30d peak TPS from PostgresDB")?; + + let epoch_peak_tps = StoredEpochPeakTps { + epoch, + peak_tps: epoch_tps.peak_tps, + peak_tps_30d: tps_30d.peak_tps, + }; + transactional_blocking_with_retry!( + &self.blocking_cp, + |conn| { + diesel::insert_into(epoch_peak_tps::table) + .values(epoch_peak_tps.clone()) + .on_conflict_do_nothing() + .execute(conn) + }, + Duration::from_secs(10) + ) + .context("Failed persisting epoch peak TPS to PostgresDB.")?; + Ok(()) + } + + async fn get_address_metrics_last_processed_tx_seq(&self) -> IndexerResult> { + let last_processed_tx_seq = read_only_blocking!(&self.blocking_cp, |conn| { + active_addresses::dsl::active_addresses + .order(active_addresses::dsl::last_appearance_tx.desc()) + .select((active_addresses::dsl::last_appearance_tx,)) + .first::(conn) + .optional() + }) + .context("Failed to read address metrics last processed tx sequence.")?; + Ok(last_processed_tx_seq) + } + + fn persist_addresses_in_tx_range( + &self, + start_tx_seq: i64, + end_tx_seq: i64, + ) -> IndexerResult<()> { + let address_persist_query = construct_address_persisting_query(start_tx_seq, end_tx_seq); + transactional_blocking_with_retry!( + &self.blocking_cp, + |conn| { + diesel::sql_query(address_persist_query.clone()).execute(conn)?; + Ok::<(), IndexerError>(()) + }, + Duration::from_secs(10) + ) + .context("Failed persisting addresses to PostgresDB")?; + Ok(()) + } + + fn persist_active_addresses_in_tx_range( + &self, + start_tx_seq: i64, + end_tx_seq: i64, + ) -> IndexerResult<()> { + let active_address_persist_query = + construct_active_address_persisting_query(start_tx_seq, end_tx_seq); + transactional_blocking_with_retry!( + &self.blocking_cp, + |conn| { + diesel::sql_query(active_address_persist_query.clone()).execute(conn)?; + Ok::<(), IndexerError>(()) + }, + Duration::from_secs(10) + ) + .context("Failed persisting active addresses to PostgresDB")?; + Ok(()) + } + + async fn calculate_and_persist_address_metrics(&self, checkpoint: i64) -> IndexerResult<()> { + let mut checkpoint_opt = self + .get_checkpoints_in_range(checkpoint, checkpoint + 1) + .await? + .pop(); + while checkpoint_opt.is_none() { + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + checkpoint_opt = self + .get_checkpoints_in_range(checkpoint, checkpoint + 1) + .await? + .pop(); + } + let checkpoint = checkpoint_opt.unwrap(); + let cp_timestamp_ms = checkpoint.timestamp_ms; + let addr_count = read_only_blocking!(&self.blocking_cp, |conn| { + addresses::dsl::addresses + .filter(addresses::first_appearance_time.le(cp_timestamp_ms)) + .count() + .get_result::(conn) + })?; + let active_addr_count = read_only_blocking!(&self.blocking_cp, |conn| { + active_addresses::dsl::active_addresses + .filter(active_addresses::first_appearance_time.le(cp_timestamp_ms)) + .count() + .get_result::(conn) + })?; + let time_one_day_ago = cp_timestamp_ms - 1000 * 60 * 60 * 24; + let daily_active_addresses = read_only_blocking!(&self.blocking_cp, |conn| { + active_addresses::dsl::active_addresses + .filter(active_addresses::first_appearance_time.le(cp_timestamp_ms)) + .filter(active_addresses::last_appearance_time.gt(time_one_day_ago)) + .select(count(active_addresses::address)) + .first(conn) + })?; + let address_metrics_to_commit = StoredAddressMetrics { + checkpoint: checkpoint.sequence_number, + epoch: checkpoint.epoch, + timestamp_ms: checkpoint.timestamp_ms, + cumulative_addresses: addr_count, + cumulative_active_addresses: active_addr_count, + daily_active_addresses, + }; + transactional_blocking_with_retry!( + &self.blocking_cp, + |conn| { + diesel::insert_into(address_metrics::table) + .values(address_metrics_to_commit.clone()) + .on_conflict_do_nothing() + .execute(conn) + }, + Duration::from_secs(60) + ) + .context("Failed persisting address metrics to PostgresDB")?; + Ok(()) + } + + async fn get_latest_move_call_tx_seq(&self) -> IndexerResult> { + let last_processed_tx_seq = read_only_blocking!(&self.blocking_cp, |conn| { + move_calls::dsl::move_calls + .order(move_calls::dsl::transaction_sequence_number.desc()) + .select((move_calls::dsl::transaction_sequence_number,)) + .first::(conn) + .optional() + }) + .unwrap_or_default(); + Ok(last_processed_tx_seq) + } + + async fn get_latest_move_call_metrics(&self) -> IndexerResult> { + let latest_move_call_metrics = read_only_blocking!(&self.blocking_cp, |conn| { + move_call_metrics::dsl::move_call_metrics + .order(move_call_metrics::epoch.desc()) + .first::(conn) + .optional() + }) + .unwrap_or_default(); + Ok(latest_move_call_metrics.map(|m| m.into())) + } + + fn persist_move_calls_in_tx_range( + &self, + start_tx_seq: i64, + end_tx_seq: i64, + ) -> IndexerResult<()> { + let move_call_persist_query = construct_move_call_persist_query(start_tx_seq, end_tx_seq); + transactional_blocking_with_retry!( + &self.blocking_cp, + |conn| { + diesel::sql_query(move_call_persist_query.clone()).execute(conn)?; + Ok::<(), IndexerError>(()) + }, + Duration::from_secs(10) + ) + .context("Failed persisting move calls to PostgresDB")?; + Ok(()) + } + + async fn calculate_and_persist_move_call_metrics(&self, epoch: i64) -> IndexerResult<()> { + let move_call_query_3d = build_move_call_metric_query(epoch, 3); + let move_call_query_7d = build_move_call_metric_query(epoch, 7); + let move_call_query_30d = build_move_call_metric_query(epoch, 30); + + let mut calculate_tasks = vec![]; + let blocking_cp_3d = self.blocking_cp.clone(); + calculate_tasks.push(tokio::task::spawn_blocking(move || { + read_only_blocking!(&blocking_cp_3d, |conn| { + diesel::sql_query(move_call_query_3d).get_results::(conn) + }) + })); + let blocking_cp_7d = self.blocking_cp.clone(); + calculate_tasks.push(tokio::task::spawn_blocking(move || { + read_only_blocking!(&blocking_cp_7d, |conn| { + diesel::sql_query(move_call_query_7d).get_results::(conn) + }) + })); + let blocking_cp_30d = self.blocking_cp.clone(); + calculate_tasks.push(tokio::task::spawn_blocking(move || { + read_only_blocking!(&blocking_cp_30d, |conn| { + diesel::sql_query(move_call_query_30d).get_results::(conn) + }) + })); + let chained = futures::future::join_all(calculate_tasks) + .await + .into_iter() + .collect::, _>>() + .tap_err(|e| { + error!("Error joining move call calculation tasks: {:?}", e); + })? + .into_iter() + .collect::, _>>() + .tap_err(|e| { + error!("Error calculating move call metrics: {:?}", e); + })? + .into_iter() + .flatten() + .collect::>(); + + let move_call_metrics: Vec = chained + .into_iter() + .filter_map(|queried_move_metrics| { + let package = ObjectID::from_bytes(queried_move_metrics.move_package.clone()).ok(); + let package_str = match package { + Some(p) => p.to_canonical_string(/* with_prefix */ true), + None => { + tracing::error!( + "Failed to parse move package ID: {:?}", + queried_move_metrics.move_package + ); + return None; + } + }; + Some(StoredMoveCallMetrics { + id: None, + epoch, + day: queried_move_metrics.day, + move_package: package_str, + move_module: queried_move_metrics.move_module, + move_function: queried_move_metrics.move_function, + count: queried_move_metrics.count, + }) + }) + .collect(); + + transactional_blocking_with_retry!( + &self.blocking_cp, + |conn| { + diesel::insert_into(move_call_metrics::table) + .values(move_call_metrics.clone()) + .on_conflict_do_nothing() + .execute(conn) + }, + Duration::from_secs(60) + ) + .context("Failed persisting move call metrics to PostgresDB")?; + Ok(()) + } +} + +fn construct_checkpoint_tx_count_query(start_checkpoint: i64, end_checkpoint: i64) -> String { + format!( + "With filtered_txns AS ( + SELECT + t.checkpoint_sequence_number, + c.epoch, + t.timestamp_ms, + t.success_command_count + FROM transactions t + LEFT JOIN checkpoints c + ON t.checkpoint_sequence_number = c.sequence_number + WHERE t.checkpoint_sequence_number >= {} AND t.checkpoint_sequence_number < {} + ) + INSERT INTO tx_count_metrics + SELECT + checkpoint_sequence_number, + epoch, + MAX(timestamp_ms) AS timestamp_ms, + COUNT(*) AS total_transaction_blocks, + SUM(CASE WHEN success_command_count > 0 THEN 1 ELSE 0 END) AS total_successful_transaction_blocks, + SUM(success_command_count) AS total_successful_transactions + FROM filtered_txns + GROUP BY checkpoint_sequence_number, epoch ORDER BY checkpoint_sequence_number + ON CONFLICT (checkpoint_sequence_number) DO NOTHING; + ", start_checkpoint, end_checkpoint + ) +} + +fn construct_peak_tps_query(epoch: i64, offset: i64) -> String { + format!( + "WITH filtered_checkpoints AS ( + SELECT + MAX(checkpoint_sequence_number) AS checkpoint_sequence_number, + SUM(total_successful_transactions) AS total_successful_transactions, + timestamp_ms + FROM + tx_count_metrics + WHERE epoch > ({} - {}) AND epoch <= {} + GROUP BY + timestamp_ms + ), + tps_data AS ( + SELECT + checkpoint_sequence_number, + total_successful_transactions, + timestamp_ms - LAG(timestamp_ms) OVER (ORDER BY timestamp_ms) AS time_diff + FROM + filtered_checkpoints + ) + SELECT + MAX(total_successful_transactions * 1000.0 / time_diff)::float8 as peak_tps + FROM + tps_data + WHERE + time_diff IS NOT NULL; + ", + epoch, offset, epoch + ) +} + +fn construct_address_persisting_query(start_tx_seq: i64, end_tx_seq: i64) -> String { + format!( + "WITH senders AS ( + SELECT + s.sender AS address, + s.tx_sequence_number, + t.timestamp_ms + FROM tx_senders s + JOIN transactions t + ON s.tx_sequence_number = t.tx_sequence_number + WHERE s.tx_sequence_number >= {} AND s.tx_sequence_number < {} + ), + recipients AS ( + SELECT + r.recipient AS address, + r.tx_sequence_number, + t.timestamp_ms + FROM tx_recipients r + JOIN transactions t + ON r.tx_sequence_number = t.tx_sequence_number + WHERE r.tx_sequence_number >= {} AND r.tx_sequence_number < {} + ), + union_address AS ( + SELECT + address, + MIN(tx_sequence_number) as first_seq, + MIN(timestamp_ms) AS first_timestamp, + MAX(tx_sequence_number) as last_seq, + MAX(timestamp_ms) AS last_timestamp + FROM recipients GROUP BY address + UNION ALL + SELECT + address, + MIN(tx_sequence_number) as first_seq, + MIN(timestamp_ms) AS first_timestamp, + MAX(tx_sequence_number) as last_seq, + MAX(timestamp_ms) AS last_timestamp + FROM senders GROUP BY address + ) + INSERT INTO addresses + SELECT + address, + MIN(first_seq) AS first_appearance_tx, + MIN(first_timestamp) AS first_appearance_time, + MAX(last_seq) AS last_appearance_tx, + MAX(last_timestamp) AS last_appearance_time + FROM union_address + GROUP BY address + ON CONFLICT (address) DO UPDATE + SET + last_appearance_tx = GREATEST(EXCLUDED.last_appearance_tx, addresses.last_appearance_tx), + last_appearance_time = GREATEST(EXCLUDED.last_appearance_time, addresses.last_appearance_time); + ", + start_tx_seq, end_tx_seq, start_tx_seq, end_tx_seq + ) +} + +fn construct_active_address_persisting_query(start_tx_seq: i64, end_tx_seq: i64) -> String { + format!( + "WITH senders AS ( + SELECT + s.sender AS address, + s.tx_sequence_number, + t.timestamp_ms + FROM tx_senders s + JOIN transactions t + ON s.tx_sequence_number = t.tx_sequence_number + WHERE s.tx_sequence_number >= {} AND s.tx_sequence_number < {} + ) + INSERT INTO active_addresses + SELECT + address, + MIN(tx_sequence_number) AS first_appearance_tx, + MIN(timestamp_ms) AS first_appearance_time, + MAX(tx_sequence_number) AS last_appearance_tx, + MAX(timestamp_ms) AS last_appearance_time + FROM senders + GROUP BY address + ON CONFLICT (address) DO UPDATE + SET + last_appearance_tx = GREATEST(EXCLUDED.last_appearance_tx, active_addresses.last_appearance_tx), + last_appearance_time = GREATEST(EXCLUDED.last_appearance_time, active_addresses.last_appearance_time); + ", + start_tx_seq, end_tx_seq + ) +} + +fn construct_move_call_persist_query(start_tx_seq: i64, end_tx_seq: i64) -> String { + format!( + "INSERT INTO move_calls + SELECT + m.tx_sequence_number AS transaction_sequence_number, + c.sequence_number AS checkpoint_sequence_number, + c.epoch AS epoch, + m.package AS move_package, + m.module AS move_module, + m.func AS move_function + FROM tx_calls m + INNER JOIN transactions t + ON m.tx_sequence_number = t.tx_sequence_number + INNER JOIN checkpoints c + ON t.checkpoint_sequence_number = c.sequence_number + WHERE m.tx_sequence_number >= {} AND m.tx_sequence_number < {} + ON CONFLICT (transaction_sequence_number, move_package, move_module, move_function) DO NOTHING; + ", + start_tx_seq, end_tx_seq + ) +} \ No newline at end of file From ea82f49165bea1e60ae3b64fb18a9f022b95b6f8 Mon Sep 17 00:00:00 2001 From: samuel_rufi Date: Wed, 14 Aug 2024 17:01:55 +0200 Subject: [PATCH 03/15] refactor(iota-indexer): fix schema usages --- .../src/handlers/address_metrics_processor.rs | 10 +- crates/iota-indexer/src/handlers/mod.rs | 6 +- .../handlers/move_call_metrics_processor.rs | 10 +- .../src/handlers/network_metrics_processor.rs | 14 +-- .../src/handlers/processor_orchestrator.rs | 22 ++-- crates/iota-indexer/src/indexer.rs | 5 +- crates/iota-indexer/src/indexer_reader.rs | 14 ++- crates/iota-indexer/src/main.rs | 4 +- .../src/models/address_metrics.rs | 6 +- .../src/models/move_call_metrics.rs | 21 ++-- .../src/models/network_metrics.rs | 11 +- .../src/models/tx_count_metrics.rs | 2 +- .../src/store/indexer_analytics_store.rs | 22 ++-- crates/iota-indexer/src/store/mod.rs | 2 +- .../src/store/pg_indexer_analytical_store.rs | 102 +++++++++--------- 15 files changed, 128 insertions(+), 123 deletions(-) diff --git a/crates/iota-indexer/src/handlers/address_metrics_processor.rs b/crates/iota-indexer/src/handlers/address_metrics_processor.rs index 1f473de6565..02665a74f97 100644 --- a/crates/iota-indexer/src/handlers/address_metrics_processor.rs +++ b/crates/iota-indexer/src/handlers/address_metrics_processor.rs @@ -5,9 +5,7 @@ use tap::tap::TapFallible; use tracing::{error, info}; -use crate::metrics::IndexerMetrics; -use crate::store::IndexerAnalyticalStore; -use crate::types::IndexerResult; +use crate::{metrics::IndexerMetrics, store::IndexerAnalyticalStore, types::IndexerResult}; const ADDRESS_PROCESSOR_BATCH_SIZE: usize = 80000; const PARALLELISM: usize = 10; @@ -20,8 +18,8 @@ pub struct AddressMetricsProcessor { } impl AddressMetricsProcessor - where - S: IndexerAnalyticalStore + Clone + Sync + Send + 'static, +where + S: IndexerAnalyticalStore + Clone + Sync + Send + 'static, { pub fn new(store: S, metrics: IndexerMetrics) -> AddressMetricsProcessor { let address_processor_batch_size = std::env::var("ADDRESS_PROCESSOR_BATCH_SIZE") @@ -121,4 +119,4 @@ impl AddressMetricsProcessor ); } } -} \ No newline at end of file +} diff --git a/crates/iota-indexer/src/handlers/mod.rs b/crates/iota-indexer/src/handlers/mod.rs index b7acbddead7..2f3fb5d553f 100644 --- a/crates/iota-indexer/src/handlers/mod.rs +++ b/crates/iota-indexer/src/handlers/mod.rs @@ -12,14 +12,14 @@ use crate::{ }, }; +pub mod address_metrics_processor; pub mod checkpoint_handler; pub mod committer; -pub mod objects_snapshot_processor; -pub mod tx_processor; -pub mod address_metrics_processor; pub mod move_call_metrics_processor; pub mod network_metrics_processor; +pub mod objects_snapshot_processor; pub mod processor_orchestrator; +pub mod tx_processor; #[derive(Debug)] pub struct CheckpointDataToCommit { diff --git a/crates/iota-indexer/src/handlers/move_call_metrics_processor.rs b/crates/iota-indexer/src/handlers/move_call_metrics_processor.rs index a631507f180..721f58a3c45 100644 --- a/crates/iota-indexer/src/handlers/move_call_metrics_processor.rs +++ b/crates/iota-indexer/src/handlers/move_call_metrics_processor.rs @@ -5,9 +5,7 @@ use tap::tap::TapFallible; use tracing::{error, info}; -use crate::metrics::IndexerMetrics; -use crate::store::IndexerAnalyticalStore; -use crate::types::IndexerResult; +use crate::{metrics::IndexerMetrics, store::IndexerAnalyticalStore, types::IndexerResult}; const MOVE_CALL_PROCESSOR_BATCH_SIZE: usize = 80000; const PARALLELISM: usize = 10; @@ -20,8 +18,8 @@ pub struct MoveCallMetricsProcessor { } impl MoveCallMetricsProcessor - where - S: IndexerAnalyticalStore + Clone + Sync + Send + 'static, +where + S: IndexerAnalyticalStore + Clone + Sync + Send + 'static, { pub fn new(store: S, metrics: IndexerMetrics) -> MoveCallMetricsProcessor { let move_call_processor_batch_size = std::env::var("MOVE_CALL_PROCESSOR_BATCH_SIZE") @@ -110,4 +108,4 @@ impl MoveCallMetricsProcessor last_processed_epoch = end_epoch - 1; } } -} \ No newline at end of file +} diff --git a/crates/iota-indexer/src/handlers/network_metrics_processor.rs b/crates/iota-indexer/src/handlers/network_metrics_processor.rs index e046ef8c84c..bee9937d8cf 100644 --- a/crates/iota-indexer/src/handlers/network_metrics_processor.rs +++ b/crates/iota-indexer/src/handlers/network_metrics_processor.rs @@ -5,10 +5,10 @@ use tap::tap::TapFallible; use tracing::{error, info}; -use crate::errors::IndexerError; -use crate::metrics::IndexerMetrics; -use crate::store::IndexerAnalyticalStore; -use crate::types::IndexerResult; +use crate::{ + errors::IndexerError, metrics::IndexerMetrics, store::IndexerAnalyticalStore, + types::IndexerResult, +}; const NETWORK_METRICS_PROCESSOR_BATCH_SIZE: usize = 10; const PARALLELISM: usize = 1; @@ -21,8 +21,8 @@ pub struct NetworkMetricsProcessor { } impl NetworkMetricsProcessor - where - S: IndexerAnalyticalStore + Clone + Sync + Send + 'static, +where + S: IndexerAnalyticalStore + Clone + Sync + Send + 'static, { pub fn new(store: S, metrics: IndexerMetrics) -> NetworkMetricsProcessor { let network_processor_metrics_batch_size = @@ -126,4 +126,4 @@ impl NetworkMetricsProcessor } } } -} \ No newline at end of file +} diff --git a/crates/iota-indexer/src/handlers/processor_orchestrator.rs b/crates/iota-indexer/src/handlers/processor_orchestrator.rs index 0c78d1c0698..67e8783d45c 100644 --- a/crates/iota-indexer/src/handlers/processor_orchestrator.rs +++ b/crates/iota-indexer/src/handlers/processor_orchestrator.rs @@ -5,12 +5,12 @@ use futures::future::try_join_all; use tracing::{error, info}; -use crate::metrics::IndexerMetrics; -use crate::store::IndexerAnalyticalStore; - -use super::address_metrics_processor::AddressMetricsProcessor; -use super::move_call_metrics_processor::MoveCallMetricsProcessor; -use super::network_metrics_processor::NetworkMetricsProcessor; +use super::{ + address_metrics_processor::AddressMetricsProcessor, + move_call_metrics_processor::MoveCallMetricsProcessor, + network_metrics_processor::NetworkMetricsProcessor, +}; +use crate::{metrics::IndexerMetrics, store::IndexerAnalyticalStore}; pub struct ProcessorOrchestrator { store: S, @@ -18,8 +18,8 @@ pub struct ProcessorOrchestrator { } impl ProcessorOrchestrator - where - S: IndexerAnalyticalStore + Clone + Send + Sync + 'static, +where + S: IndexerAnalyticalStore + Clone + Send + Sync + 'static, { pub fn new(store: S, metrics: IndexerMetrics) -> Self { Self { store, metrics } @@ -77,7 +77,7 @@ impl ProcessorOrchestrator addr_metrics_handle, move_call_metrics_handle, ]) - .await - .expect("Processor orchestrator should not run into errors."); + .await + .expect("Processor orchestrator should not run into errors."); } -} \ No newline at end of file +} diff --git a/crates/iota-indexer/src/indexer.rs b/crates/iota-indexer/src/indexer.rs index 3c6296aac64..d1ba3d4a638 100644 --- a/crates/iota-indexer/src/indexer.rs +++ b/crates/iota-indexer/src/indexer.rs @@ -16,14 +16,13 @@ use crate::{ handlers::{ checkpoint_handler::new_handlers, objects_snapshot_processor::{ObjectsSnapshotProcessor, SnapshotLagConfig}, + processor_orchestrator::ProcessorOrchestrator, }, indexer_reader::IndexerReader, metrics::IndexerMetrics, - store::IndexerStore, + store::{IndexerStore, PgIndexerAnalyticalStore}, IndexerConfig, }; -use crate::handlers::processor_orchestrator::ProcessorOrchestrator; -use crate::store::PgIndexerAnalyticalStore; const DOWNLOAD_QUEUE_SIZE: usize = 200; diff --git a/crates/iota-indexer/src/indexer_reader.rs b/crates/iota-indexer/src/indexer_reader.rs index bd84ca74754..827c6c9f5d7 100644 --- a/crates/iota-indexer/src/indexer_reader.rs +++ b/crates/iota-indexer/src/indexer_reader.rs @@ -15,11 +15,11 @@ use diesel::{ }; use fastcrypto::encoding::{Encoding, Hex}; use iota_json_rpc_types::{ - Balance, CheckpointId, Coin as IotaCoin, DisplayFieldsResponse, EpochInfo, EventFilter, - IotaCoinMetadata, IotaEvent, IotaObjectDataFilter, IotaTransactionBlockEffects, - IotaTransactionBlockEffectsAPI, IotaTransactionBlockResponse, TransactionFilter, + AddressMetrics, Balance, CheckpointId, Coin as IotaCoin, DisplayFieldsResponse, EpochInfo, + EventFilter, IotaCoinMetadata, IotaEvent, IotaObjectDataFilter, IotaTransactionBlockEffects, + IotaTransactionBlockEffectsAPI, IotaTransactionBlockResponse, MoveCallMetrics, + MoveFunctionName, NetworkMetrics, TransactionFilter, }; - use iota_types::{ balance::Supply, base_types::{IotaAddress, ObjectID, ObjectRef, SequenceNumber, VersionNumber}, @@ -40,17 +40,21 @@ use crate::{ db::{PgConnectionConfig, PgConnectionPoolConfig, PgPoolConnection}, errors::IndexerError, models::{ + address_metrics::StoredAddressMetrics, checkpoints::StoredCheckpoint, display::StoredDisplay, epoch::StoredEpochInfo, events::StoredEvent, + move_call_metrics::QueriedMoveCallMetrics, + network_metrics::StoredNetworkMetrics, objects::{CoinBalance, ObjectRefColumn, StoredObject}, packages::StoredPackage, transactions::StoredTransaction, tx_indices::TxSequenceNumber, }, schema::{ - checkpoints, display, epochs, events, objects, objects_snapshot, packages, transactions, + address_metrics, checkpoints, display, epochs, events, move_call_metrics, objects, + objects_snapshot, packages, transactions, }, types::{IndexerResult, OwnerType}, }; diff --git a/crates/iota-indexer/src/main.rs b/crates/iota-indexer/src/main.rs index a5c1e895aba..af117c1813d 100644 --- a/crates/iota-indexer/src/main.rs +++ b/crates/iota-indexer/src/main.rs @@ -8,9 +8,7 @@ use iota_indexer::{ errors::IndexerError, indexer::Indexer, metrics::{start_prometheus_server, IndexerMetrics}, - store::{ - PgIndexerAnalyticalStore, PgIndexerStore, - }, + store::{PgIndexerAnalyticalStore, PgIndexerStore}, IndexerConfig, }; use tracing::{error, info}; diff --git a/crates/iota-indexer/src/models/address_metrics.rs b/crates/iota-indexer/src/models/address_metrics.rs index bc3dfbdf0d3..a6e78256f2e 100644 --- a/crates/iota-indexer/src/models/address_metrics.rs +++ b/crates/iota-indexer/src/models/address_metrics.rs @@ -4,9 +4,7 @@ use std::collections::HashMap; -use diesel::prelude::*; -use diesel::sql_types::BigInt; - +use diesel::{prelude::*, sql_types::BigInt}; use iota_json_rpc_types::AddressMetrics; use crate::schema::{active_addresses, address_metrics, addresses}; @@ -109,4 +107,4 @@ pub fn dedup_addresses(addrs_to_commit: Vec) -> Vec String { GROUP BY move_package, move_module, move_function ORDER BY count DESC LIMIT 10;", epoch, days, epoch - days) -} \ No newline at end of file +} diff --git a/crates/iota-indexer/src/models/network_metrics.rs b/crates/iota-indexer/src/models/network_metrics.rs index 7fa2d1c5219..55681e0aec4 100644 --- a/crates/iota-indexer/src/models/network_metrics.rs +++ b/crates/iota-indexer/src/models/network_metrics.rs @@ -2,10 +2,11 @@ // Modifications Copyright (c) 2024 IOTA Stiftung // SPDX-License-Identifier: Apache-2.0 -use diesel::prelude::*; -use diesel::sql_types::{BigInt, Double, Float8}; - -use sui_json_rpc_types::NetworkMetrics; +use diesel::{ + prelude::*, + sql_types::{BigInt, Double, Float8}, +}; +use iota_json_rpc_types::NetworkMetrics; use crate::schema::epoch_peak_tps; @@ -63,4 +64,4 @@ impl From for NetworkMetrics { pub struct Tps { #[diesel(sql_type = Float8)] pub peak_tps: f64, -} \ No newline at end of file +} diff --git a/crates/iota-indexer/src/models/tx_count_metrics.rs b/crates/iota-indexer/src/models/tx_count_metrics.rs index 388eff9bb89..b645edb8aa3 100644 --- a/crates/iota-indexer/src/models/tx_count_metrics.rs +++ b/crates/iota-indexer/src/models/tx_count_metrics.rs @@ -28,4 +28,4 @@ impl Default for StoredTxCountMetrics { total_successful_transactions: -1, } } -} \ No newline at end of file +} diff --git a/crates/iota-indexer/src/store/indexer_analytics_store.rs b/crates/iota-indexer/src/store/indexer_analytics_store.rs index 0e964a3b931..ecfd107a8f3 100644 --- a/crates/iota-indexer/src/store/indexer_analytics_store.rs +++ b/crates/iota-indexer/src/store/indexer_analytics_store.rs @@ -4,15 +4,19 @@ use async_trait::async_trait; -use crate::models::checkpoints::StoredCheckpoint; -use crate::models::move_call_metrics::StoredMoveCallMetrics; -use crate::models::network_metrics::StoredEpochPeakTps; -use crate::models::transactions::{ - StoredTransaction, StoredTransactionCheckpoint, StoredTransactionSuccessCommandCount, - StoredTransactionTimestamp, TxSeq, +use crate::{ + models::{ + checkpoints::StoredCheckpoint, + move_call_metrics::StoredMoveCallMetrics, + network_metrics::StoredEpochPeakTps, + transactions::{ + StoredTransaction, StoredTransactionCheckpoint, StoredTransactionSuccessCommandCount, + StoredTransactionTimestamp, TxSeq, + }, + tx_count_metrics::StoredTxCountMetrics, + }, + types::IndexerResult, }; -use crate::models::tx_count_metrics::StoredTxCountMetrics; -use crate::types::IndexerResult; #[async_trait] pub trait IndexerAnalyticalStore { @@ -74,4 +78,4 @@ pub trait IndexerAnalyticalStore { end_tx_seq: i64, ) -> IndexerResult<()>; async fn calculate_and_persist_move_call_metrics(&self, epoch: i64) -> IndexerResult<()>; -} \ No newline at end of file +} diff --git a/crates/iota-indexer/src/store/mod.rs b/crates/iota-indexer/src/store/mod.rs index cd233140c2f..9214ec5e34a 100644 --- a/crates/iota-indexer/src/store/mod.rs +++ b/crates/iota-indexer/src/store/mod.rs @@ -2,7 +2,7 @@ // Modifications Copyright (c) 2024 IOTA Stiftung // SPDX-License-Identifier: Apache-2.0 -pub(crate) use indexer_analytical_store::*; +pub(crate) use indexer_analytics_store::*; pub(crate) use indexer_store::*; pub use pg_indexer_analytical_store::PgIndexerAnalyticalStore; pub use pg_indexer_store::PgIndexerStore; diff --git a/crates/iota-indexer/src/store/pg_indexer_analytical_store.rs b/crates/iota-indexer/src/store/pg_indexer_analytical_store.rs index e9fa3a4bc02..5b143e4ad23 100644 --- a/crates/iota-indexer/src/store/pg_indexer_analytical_store.rs +++ b/crates/iota-indexer/src/store/pg_indexer_analytical_store.rs @@ -2,38 +2,40 @@ // Modifications Copyright (c) 2024 IOTA Stiftung // SPDX-License-Identifier: Apache-2.0 +use core::result::Result::Ok; use std::time::Duration; -use tap::tap::TapFallible; -use tracing::{error, info}; use async_trait::async_trait; -use core::result::Result::Ok; -use diesel::dsl::count; -use diesel::{ExpressionMethods, OptionalExtension}; -use diesel::{QueryDsl, RunQueryDsl}; -use sui_types::base_types::ObjectID; - -use crate::db::PgConnectionPool; -use crate::errors::{Context, IndexerError}; -use crate::models::address_metrics::StoredAddressMetrics; -use crate::models::checkpoints::StoredCheckpoint; -use crate::models::move_call_metrics::{ - build_move_call_metric_query, QueriedMoveCallMetrics, QueriedMoveMetrics, StoredMoveCallMetrics, -}; -use crate::models::network_metrics::{StoredEpochPeakTps, Tps}; -use crate::models::transactions::{ - StoredTransaction, StoredTransactionCheckpoint, StoredTransactionSuccessCommandCount, - StoredTransactionTimestamp, TxSeq, -}; -use crate::models::tx_count_metrics::StoredTxCountMetrics; -use crate::schema::{ - active_addresses, address_metrics, addresses, checkpoints, epoch_peak_tps, move_call_metrics, - move_calls, transactions, tx_count_metrics, -}; -use crate::store::diesel_macro::{read_only_blocking, transactional_blocking_with_retry}; -use crate::types::IndexerResult; +use diesel::{dsl::count, ExpressionMethods, OptionalExtension, QueryDsl, RunQueryDsl}; +use iota_types::base_types::ObjectID; +use tap::tap::TapFallible; +use tracing::{error, info}; use super::IndexerAnalyticalStore; +use crate::{ + db::PgConnectionPool, + errors::{Context, IndexerError}, + models::{ + address_metrics::StoredAddressMetrics, + checkpoints::StoredCheckpoint, + move_call_metrics::{ + build_move_call_metric_query, QueriedMoveCallMetrics, QueriedMoveMetrics, + StoredMoveCallMetrics, + }, + network_metrics::{StoredEpochPeakTps, Tps}, + transactions::{ + StoredTransaction, StoredTransactionCheckpoint, StoredTransactionSuccessCommandCount, + StoredTransactionTimestamp, TxSeq, + }, + tx_count_metrics::StoredTxCountMetrics, + }, + schema::{ + active_addresses, address_metrics, addresses, checkpoints, epoch_peak_tps, + move_call_metrics, move_calls, transactions, tx_count_metrics, + }, + store::diesel_macro::{read_only_blocking, transactional_blocking_with_retry}, + types::IndexerResult, +}; #[derive(Clone)] pub struct PgIndexerAnalyticalStore { @@ -55,7 +57,7 @@ impl IndexerAnalyticalStore for PgIndexerAnalyticalStore { .first::(conn) .optional() }) - .context("Failed reading latest checkpoint from PostgresDB")?; + .context("Failed reading latest checkpoint from PostgresDB")?; Ok(latest_cp) } @@ -66,7 +68,7 @@ impl IndexerAnalyticalStore for PgIndexerAnalyticalStore { .first::(conn) .optional() }) - .context("Failed reading latest transaction from PostgresDB")?; + .context("Failed reading latest transaction from PostgresDB")?; Ok(latest_tx) } @@ -82,7 +84,7 @@ impl IndexerAnalyticalStore for PgIndexerAnalyticalStore { .order(checkpoints::sequence_number.asc()) .load::(conn) }) - .context("Failed reading checkpoints from PostgresDB")?; + .context("Failed reading checkpoints from PostgresDB")?; Ok(cps) } @@ -102,7 +104,7 @@ impl IndexerAnalyticalStore for PgIndexerAnalyticalStore { )) .load::(conn) }) - .context("Failed reading transaction timestamps from PostgresDB")?; + .context("Failed reading transaction timestamps from PostgresDB")?; Ok(tx_timestamps) } @@ -122,7 +124,7 @@ impl IndexerAnalyticalStore for PgIndexerAnalyticalStore { )) .load::(conn) }) - .context("Failed reading transaction checkpoints from PostgresDB")?; + .context("Failed reading transaction checkpoints from PostgresDB")?; Ok(tx_checkpoints) } @@ -144,7 +146,7 @@ impl IndexerAnalyticalStore for PgIndexerAnalyticalStore { )) .load::(conn) }) - .context("Failed reading transaction success command counts from PostgresDB")?; + .context("Failed reading transaction success command counts from PostgresDB")?; Ok(tx_success_cmd_counts) } async fn get_tx(&self, tx_sequence_number: i64) -> IndexerResult> { @@ -154,7 +156,7 @@ impl IndexerAnalyticalStore for PgIndexerAnalyticalStore { .first::(conn) .optional() }) - .context("Failed reading transaction from PostgresDB")?; + .context("Failed reading transaction from PostgresDB")?; Ok(tx) } @@ -165,7 +167,7 @@ impl IndexerAnalyticalStore for PgIndexerAnalyticalStore { .first::(conn) .optional() }) - .context("Failed reading checkpoint from PostgresDB")?; + .context("Failed reading checkpoint from PostgresDB")?; Ok(cp) } @@ -176,7 +178,7 @@ impl IndexerAnalyticalStore for PgIndexerAnalyticalStore { .first::(conn) .optional() }) - .context("Failed reading latest tx count metrics from PostgresDB")?; + .context("Failed reading latest tx count metrics from PostgresDB")?; Ok(latest_tx_count) } @@ -187,7 +189,7 @@ impl IndexerAnalyticalStore for PgIndexerAnalyticalStore { .first::(conn) .optional() }) - .context("Failed reading latest epoch peak TPS from PostgresDB")?; + .context("Failed reading latest epoch peak TPS from PostgresDB")?; Ok(latest_network_metrics) } @@ -206,7 +208,7 @@ impl IndexerAnalyticalStore for PgIndexerAnalyticalStore { }, Duration::from_secs(10) ) - .context("Failed persisting tx count metrics to PostgresDB")?; + .context("Failed persisting tx count metrics to PostgresDB")?; info!("Persisted tx count metrics for cp {}", start_checkpoint); Ok(()) } @@ -219,13 +221,13 @@ impl IndexerAnalyticalStore for PgIndexerAnalyticalStore { diesel::sql_query(epoch_peak_tps_query), conn )) - .context("Failed reading epoch peak TPS from PostgresDB")?; + .context("Failed reading epoch peak TPS from PostgresDB")?; let tps_30d: Tps = read_only_blocking!(&self.blocking_cp, |conn| diesel::RunQueryDsl::get_result( diesel::sql_query(peak_tps_30d_query), conn )) - .context("Failed reading 30d peak TPS from PostgresDB")?; + .context("Failed reading 30d peak TPS from PostgresDB")?; let epoch_peak_tps = StoredEpochPeakTps { epoch, @@ -242,7 +244,7 @@ impl IndexerAnalyticalStore for PgIndexerAnalyticalStore { }, Duration::from_secs(10) ) - .context("Failed persisting epoch peak TPS to PostgresDB.")?; + .context("Failed persisting epoch peak TPS to PostgresDB.")?; Ok(()) } @@ -254,7 +256,7 @@ impl IndexerAnalyticalStore for PgIndexerAnalyticalStore { .first::(conn) .optional() }) - .context("Failed to read address metrics last processed tx sequence.")?; + .context("Failed to read address metrics last processed tx sequence.")?; Ok(last_processed_tx_seq) } @@ -272,7 +274,7 @@ impl IndexerAnalyticalStore for PgIndexerAnalyticalStore { }, Duration::from_secs(10) ) - .context("Failed persisting addresses to PostgresDB")?; + .context("Failed persisting addresses to PostgresDB")?; Ok(()) } @@ -291,7 +293,7 @@ impl IndexerAnalyticalStore for PgIndexerAnalyticalStore { }, Duration::from_secs(10) ) - .context("Failed persisting active addresses to PostgresDB")?; + .context("Failed persisting active addresses to PostgresDB")?; Ok(()) } @@ -347,7 +349,7 @@ impl IndexerAnalyticalStore for PgIndexerAnalyticalStore { }, Duration::from_secs(60) ) - .context("Failed persisting address metrics to PostgresDB")?; + .context("Failed persisting address metrics to PostgresDB")?; Ok(()) } @@ -359,7 +361,7 @@ impl IndexerAnalyticalStore for PgIndexerAnalyticalStore { .first::(conn) .optional() }) - .unwrap_or_default(); + .unwrap_or_default(); Ok(last_processed_tx_seq) } @@ -370,7 +372,7 @@ impl IndexerAnalyticalStore for PgIndexerAnalyticalStore { .first::(conn) .optional() }) - .unwrap_or_default(); + .unwrap_or_default(); Ok(latest_move_call_metrics.map(|m| m.into())) } @@ -388,7 +390,7 @@ impl IndexerAnalyticalStore for PgIndexerAnalyticalStore { }, Duration::from_secs(10) ) - .context("Failed persisting move calls to PostgresDB")?; + .context("Failed persisting move calls to PostgresDB")?; Ok(()) } @@ -468,7 +470,7 @@ impl IndexerAnalyticalStore for PgIndexerAnalyticalStore { }, Duration::from_secs(60) ) - .context("Failed persisting move call metrics to PostgresDB")?; + .context("Failed persisting move call metrics to PostgresDB")?; Ok(()) } } @@ -640,4 +642,4 @@ fn construct_move_call_persist_query(start_tx_seq: i64, end_tx_seq: i64) -> Stri ", start_tx_seq, end_tx_seq ) -} \ No newline at end of file +} From 04568138dd124fb032c152be316c15e74a264b7d Mon Sep 17 00:00:00 2001 From: samuel_rufi Date: Fri, 16 Aug 2024 10:00:04 +0200 Subject: [PATCH 04/15] refactor(iota-indexer): redo `ExtendedApi` methods --- crates/iota-indexer/src/apis/extended_api.rs | 34 +++++++++++++++---- .../handlers/objects_snapshot_processor.rs | 10 +++++- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/crates/iota-indexer/src/apis/extended_api.rs b/crates/iota-indexer/src/apis/extended_api.rs index c08190f8179..982b2f3eac2 100644 --- a/crates/iota-indexer/src/apis/extended_api.rs +++ b/crates/iota-indexer/src/apis/extended_api.rs @@ -106,26 +106,46 @@ impl ExtendedApiServer for ExtendedApi { } async fn get_network_metrics(&self) -> RpcResult { - Err(jsonrpsee::types::error::ErrorCode::MethodNotFound.into()) + let network_metrics = self + .inner + .spawn_blocking(|this| this.get_latest_network_metrics()) + .await?; + Ok(network_metrics) } async fn get_move_call_metrics(&self) -> RpcResult { - Err(jsonrpsee::types::error::ErrorCode::MethodNotFound.into()) + let move_call_metrics = self + .inner + .spawn_blocking(|this| this.get_latest_move_call_metrics()) + .await?; + Ok(move_call_metrics) } async fn get_latest_address_metrics(&self) -> RpcResult { - Err(jsonrpsee::types::error::ErrorCode::MethodNotFound.into()) + let latest_address_metrics = self + .inner + .spawn_blocking(|this| this.get_latest_address_metrics()) + .await?; + Ok(latest_address_metrics) } - async fn get_checkpoint_address_metrics(&self, _checkpoint: u64) -> RpcResult { - Err(jsonrpsee::types::error::ErrorCode::MethodNotFound.into()) + async fn get_checkpoint_address_metrics(&self, checkpoint: u64) -> RpcResult { + let checkpoint_address_metrics = self + .inner + .spawn_blocking(move |this| this.get_checkpoint_address_metrics(checkpoint)) + .await?; + Ok(checkpoint_address_metrics) } async fn get_all_epoch_address_metrics( &self, - _descending_order: Option, + descending_order: Option, ) -> RpcResult> { - Err(jsonrpsee::types::error::ErrorCode::MethodNotFound.into()) + let all_epoch_address_metrics = self + .inner + .spawn_blocking(move |this| this.get_all_epoch_address_metrics(descending_order)) + .await?; + Ok(all_epoch_address_metrics) } async fn query_objects( diff --git a/crates/iota-indexer/src/handlers/objects_snapshot_processor.rs b/crates/iota-indexer/src/handlers/objects_snapshot_processor.rs index 41236720440..7da7903c87d 100644 --- a/crates/iota-indexer/src/handlers/objects_snapshot_processor.rs +++ b/crates/iota-indexer/src/handlers/objects_snapshot_processor.rs @@ -57,11 +57,19 @@ impl Default for SnapshotLagConfig { } } -// NOTE: "handler" +// TODO: "handler" impl ObjectsSnapshotProcessor where S: IndexerStore + Clone + Sync + Send + 'static, { + pub fn new(store: S, metrics: IndexerMetrics) -> ObjectsSnapshotProcessor { + Self { + store, + metrics, + config: SnapshotLagConfig::default(), + } + } + pub fn new_with_config( store: S, metrics: IndexerMetrics, From 72c78cae8940c570938e71682364d14d70c6537f Mon Sep 17 00:00:00 2001 From: samuel_rufi Date: Fri, 16 Aug 2024 15:35:10 +0200 Subject: [PATCH 05/15] refactor(iota-indexer): Introduce analytical-worker in docker-compose --- crates/iota-indexer/src/indexer.rs | 2 +- docker/pg-services-local/docker-compose.yaml | 26 ++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/crates/iota-indexer/src/indexer.rs b/crates/iota-indexer/src/indexer.rs index d1ba3d4a638..c7ecdf7122c 100644 --- a/crates/iota-indexer/src/indexer.rs +++ b/crates/iota-indexer/src/indexer.rs @@ -122,7 +122,7 @@ impl Indexer { metrics: IndexerMetrics, ) -> Result<(), IndexerError> { info!( - "Sui Indexer Analytical Worker (version {:?}) started...", + "Iota Indexer Analytical Worker (version {:?}) started...", env!("CARGO_PKG_VERSION") ); let mut processor_orchestrator = ProcessorOrchestrator::new(store, metrics); diff --git a/docker/pg-services-local/docker-compose.yaml b/docker/pg-services-local/docker-compose.yaml index 1d8c8413075..e387e50bd28 100644 --- a/docker/pg-services-local/docker-compose.yaml +++ b/docker/pg-services-local/docker-compose.yaml @@ -84,6 +84,32 @@ services: - postgres - indexer-sync + analytical-worker: + image: iota-indexer + container_name: analytical-worker + hostname: analytical-worker + restart: on-failure + networks: + iota-network: + environment: + - RUST_BACKTRACE=1 + - RUST_LOG=info + - RPC_WORKER_THREAD=12 + command: + - /usr/local/bin/iota-indexer + - --db-url=postgres://iota_indexer:iota_indexer@postgres:5432/iota_indexer + - --rpc-client-url=http://local-network:9000 + - --client-metric-port=9181 + - --rpc-server-port=9000 + - --analytical-worker + ports: + - "127.0.0.1:9006:9000/tcp" + - "127.0.0.1:9184:9181/tcp" + depends_on: + - local-network + - postgres + - indexer-sync + graphql-server: image: iota-graphql-rpc build: From e1a44f299b9013a13cf81c2c62b94cf378c7014d Mon Sep 17 00:00:00 2001 From: samuel_rufi Date: Tue, 20 Aug 2024 11:39:11 +0200 Subject: [PATCH 06/15] refactor(iota-benchmark, iota-core): Fix format --- crates/iota-benchmark/src/embedded_reconfig_observer.rs | 3 +-- crates/iota-core/src/checkpoints/causal_order.rs | 3 +-- crates/iota-core/src/checkpoints/checkpoint_executor/mod.rs | 3 +-- crates/iota-core/src/checkpoints/checkpoint_output.rs | 3 +-- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/crates/iota-benchmark/src/embedded_reconfig_observer.rs b/crates/iota-benchmark/src/embedded_reconfig_observer.rs index a84e45f56b4..b1ad903a11b 100644 --- a/crates/iota-benchmark/src/embedded_reconfig_observer.rs +++ b/crates/iota-benchmark/src/embedded_reconfig_observer.rs @@ -55,8 +55,7 @@ impl EmbeddedReconfigObserver { if new_epoch <= cur_epoch { trace!( cur_epoch, - new_epoch, - "Ignored Committee from a previous or current epoch", + new_epoch, "Ignored Committee from a previous or current epoch", ); return Ok(auth_agg); } diff --git a/crates/iota-core/src/checkpoints/causal_order.rs b/crates/iota-core/src/checkpoints/causal_order.rs index 0d7c6cf5270..b907ea16b17 100644 --- a/crates/iota-core/src/checkpoints/causal_order.rs +++ b/crates/iota-core/src/checkpoints/causal_order.rs @@ -181,8 +181,7 @@ impl RWLockDependencyBuilder { for dep in reads { trace!( "Assuming additional dependency when constructing checkpoint {:?} -> {:?}", - digest, - *dep + digest, *dep ); v.insert(*dep); } diff --git a/crates/iota-core/src/checkpoints/checkpoint_executor/mod.rs b/crates/iota-core/src/checkpoints/checkpoint_executor/mod.rs index 966d2ff3bb4..feebf06ec4e 100644 --- a/crates/iota-core/src/checkpoints/checkpoint_executor/mod.rs +++ b/crates/iota-core/src/checkpoints/checkpoint_executor/mod.rs @@ -740,8 +740,7 @@ async fn handle_execution_effects( if checkpoint.sequence_number > highest_seq + 1 { trace!( "Checkpoint {} is still executing. Highest executed = {}", - checkpoint.sequence_number, - highest_seq + checkpoint.sequence_number, highest_seq ); continue; } diff --git a/crates/iota-core/src/checkpoints/checkpoint_output.rs b/crates/iota-core/src/checkpoints/checkpoint_output.rs index e6712c54af0..8eb91cd22f3 100644 --- a/crates/iota-core/src/checkpoints/checkpoint_output.rs +++ b/crates/iota-core/src/checkpoints/checkpoint_output.rs @@ -124,8 +124,7 @@ impl CheckpointOutput for LogCheckpointOutput { ) -> IotaResult { trace!( "Including following transactions in checkpoint {}: {:?}", - summary.sequence_number, - contents + summary.sequence_number, contents ); info!( "Creating checkpoint {:?} at epoch {}, sequence {}, previous digest {:?}, transactions count {}, content digest {:?}, end_of_epoch_data {:?}", From 84bca01de73ff12e4567f40700c4fa6741552e8a Mon Sep 17 00:00:00 2001 From: samuel_rufi Date: Tue, 20 Aug 2024 15:10:26 +0200 Subject: [PATCH 07/15] refactor: Adjust docker-compose params and ports --- docker/pg-services-local/docker-compose.yaml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/docker/pg-services-local/docker-compose.yaml b/docker/pg-services-local/docker-compose.yaml index e387e50bd28..910fd0143db 100644 --- a/docker/pg-services-local/docker-compose.yaml +++ b/docker/pg-services-local/docker-compose.yaml @@ -84,10 +84,10 @@ services: - postgres - indexer-sync - analytical-worker: + indexer-analytics: image: iota-indexer - container_name: analytical-worker - hostname: analytical-worker + container_name: indexer-analytics + hostname: indexer-analytics restart: on-failure networks: iota-network: @@ -100,10 +100,8 @@ services: - --db-url=postgres://iota_indexer:iota_indexer@postgres:5432/iota_indexer - --rpc-client-url=http://local-network:9000 - --client-metric-port=9181 - - --rpc-server-port=9000 - --analytical-worker ports: - - "127.0.0.1:9006:9000/tcp" - "127.0.0.1:9184:9181/tcp" depends_on: - local-network From 6617dcd49adc09aa382513d0cae9dee7714167a0 Mon Sep 17 00:00:00 2001 From: samuel_rufi Date: Tue, 20 Aug 2024 15:20:17 +0200 Subject: [PATCH 08/15] refactor: Add missing documentation --- .../src/store/indexer_analytics_store.rs | 2 +- .../src/store/pg_indexer_analytical_store.rs | 1 + crates/iota-json-rpc-api/src/extended.rs | 16 +++++----- .../iota-json-rpc-types/src/iota_extended.rs | 32 ++++++++++++++++--- 4 files changed, 37 insertions(+), 14 deletions(-) diff --git a/crates/iota-indexer/src/store/indexer_analytics_store.rs b/crates/iota-indexer/src/store/indexer_analytics_store.rs index ecfd107a8f3..1a3353e347c 100644 --- a/crates/iota-indexer/src/store/indexer_analytics_store.rs +++ b/crates/iota-indexer/src/store/indexer_analytics_store.rs @@ -18,7 +18,7 @@ use crate::{ types::IndexerResult, }; -#[async_trait] +/// Provides methods to get and persist metrics. Utility methods for calculating metrics are also provided. pub trait IndexerAnalyticalStore { async fn get_latest_stored_transaction(&self) -> IndexerResult>; async fn get_latest_stored_checkpoint(&self) -> IndexerResult>; diff --git a/crates/iota-indexer/src/store/pg_indexer_analytical_store.rs b/crates/iota-indexer/src/store/pg_indexer_analytical_store.rs index 5b143e4ad23..2bd104d8c52 100644 --- a/crates/iota-indexer/src/store/pg_indexer_analytical_store.rs +++ b/crates/iota-indexer/src/store/pg_indexer_analytical_store.rs @@ -37,6 +37,7 @@ use crate::{ types::IndexerResult, }; +/// The store for the indexer analytical data. Represents a Postgres implementation of the `IndexerAnalyticalStore` trait. #[derive(Clone)] pub struct PgIndexerAnalyticalStore { blocking_cp: PgConnectionPool, diff --git a/crates/iota-json-rpc-api/src/extended.rs b/crates/iota-json-rpc-api/src/extended.rs index 221d6b7bb9a..66113e99558 100644 --- a/crates/iota-json-rpc-api/src/extended.rs +++ b/crates/iota-json-rpc-api/src/extended.rs @@ -20,11 +20,11 @@ pub trait ExtendedApi { #[method(name = "getEpochs")] async fn get_epochs( &self, - /// optional paging cursor + /// Optional paging cursor cursor: Option>, - /// maximum number of items per page + /// Maximum number of items per page limit: Option, - /// flag to return results in descending order + /// Flag to return results in descending order descending_order: Option, ) -> RpcResult; @@ -32,11 +32,11 @@ pub trait ExtendedApi { #[method(name = "getEpochMetrics")] async fn get_epoch_metrics( &self, - /// optional paging cursor + /// Optional paging cursor cursor: Option>, - /// maximum number of items per page + /// Maximum number of items per page limit: Option, - /// flag to return results in descending order + /// Flag to return results in descending order descending_order: Option, ) -> RpcResult; @@ -49,7 +49,7 @@ pub trait ExtendedApi { #[method(name = "queryObjects")] async fn query_objects( &self, - /// the objects query criteria. + /// The objects query criteria. query: IotaObjectResponseQuery, /// An optional paging cursor. If provided, the query will start from the next item after the specified cursor. Default to start from the first item if not specified. cursor: Option, @@ -61,7 +61,7 @@ pub trait ExtendedApi { #[method(name = "getNetworkMetrics")] async fn get_network_metrics(&self) -> RpcResult; - /// Return Network metrics + /// Return move call metrics #[method(name = "getMoveCallMetrics")] async fn get_move_call_metrics(&self) -> RpcResult; diff --git a/crates/iota-json-rpc-types/src/iota_extended.rs b/crates/iota-json-rpc-types/src/iota_extended.rs index db8b1e502c1..f6c5e6ff476 100644 --- a/crates/iota-json-rpc-types/src/iota_extended.rs +++ b/crates/iota-json-rpc-types/src/iota_extended.rs @@ -26,24 +26,27 @@ pub type EpochMetricsPage = Page>; #[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct EpochInfo { - /// epoch number + /// Epoch number #[schemars(with = "BigInt")] #[serde_as(as = "BigInt")] pub epoch: EpochId, - /// list of validators included in epoch + /// List of validators included in epoch pub validators: Vec, - /// count of tx in epoch + /// Count of tx in epoch #[schemars(with = "BigInt")] #[serde_as(as = "BigInt")] pub epoch_total_transactions: u64, - /// first, last checkpoint sequence numbers + /// First, last checkpoint sequence numbers #[schemars(with = "BigInt")] #[serde_as(as = "BigInt")] pub first_checkpoint_id: CheckpointSequenceNumber, + /// The timestamp when the epoch started. #[schemars(with = "BigInt")] #[serde_as(as = "BigInt")] pub epoch_start_timestamp: u64, + /// The end of epoch information. pub end_of_epoch_info: Option, + /// The reference gas price for the given epoch. pub reference_gas_price: Option, } @@ -58,23 +61,28 @@ impl EpochInfo { } } -/// a light-weight version of `EpochInfo` for faster loading +/// A light-weight version of `EpochInfo` for faster loading #[serde_as] #[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct EpochMetrics { + /// The current epoch ID. #[schemars(with = "BigInt")] #[serde_as(as = "BigInt")] pub epoch: EpochId, + /// The total number of transactions in the epoch. #[schemars(with = "BigInt")] #[serde_as(as = "BigInt")] pub epoch_total_transactions: u64, + /// The first checkpoint ID of the epoch. #[schemars(with = "BigInt")] #[serde_as(as = "BigInt")] pub first_checkpoint_id: CheckpointSequenceNumber, + /// The timestamp when the epoch started. #[schemars(with = "BigInt")] #[serde_as(as = "BigInt")] pub epoch_start_timestamp: u64, + /// The end of epoch information. pub end_of_epoch_info: Option, } @@ -155,38 +163,52 @@ pub struct NetworkMetrics { #[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct MoveCallMetrics { + /// The count of calls of each function in the last 3 days. #[schemars(with = "Vec<(MoveFunctionName, BigInt)>")] #[serde_as(as = "Vec<(_, BigInt)>")] pub rank_3_days: Vec<(MoveFunctionName, usize)>, + /// The count of calls of each function in the last 7 days. #[schemars(with = "Vec<(MoveFunctionName, BigInt)>")] #[serde_as(as = "Vec<(_, BigInt)>")] pub rank_7_days: Vec<(MoveFunctionName, usize)>, + /// The count of calls of each function in the last 30 days. #[schemars(with = "Vec<(MoveFunctionName, BigInt)>")] #[serde_as(as = "Vec<(_, BigInt)>")] pub rank_30_days: Vec<(MoveFunctionName, usize)>, } +/// Identifies a Move function. #[serde_as] #[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct MoveFunctionName { + /// The package ID to which the function belongs. pub package: ObjectID, + /// The module name to which the function belongs. #[schemars(with = "String")] #[serde_as(as = "DisplayFromStr")] pub module: Identifier, + /// The function name. #[schemars(with = "String")] #[serde_as(as = "DisplayFromStr")] pub function: Identifier, } +/// Provides metrics about the addresses. #[serde_as] #[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct AddressMetrics { + /// The checkpoint sequence number at which the metrics were computed. pub checkpoint: u64, + /// The epoch to which the checkpoint is assigned. pub epoch: u64, + /// The checkpoint timestamp. pub timestamp_ms: u64, + /// The count of sender and recipient addresses. pub cumulative_addresses: u64, + /// The count of sender addresses. pub cumulative_active_addresses: u64, + /// The count of daily unique sender addresses. pub daily_active_addresses: u64, } From 0d123a9a0b6e5487015321fc139eab5cee5f29ec Mon Sep 17 00:00:00 2001 From: Samuel Rufinatscha Date: Tue, 20 Aug 2024 15:21:03 +0200 Subject: [PATCH 09/15] refactor: Explicit export Co-authored-by: Konstantinos Demartinos --- crates/iota-indexer/src/store/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/iota-indexer/src/store/mod.rs b/crates/iota-indexer/src/store/mod.rs index 9214ec5e34a..4fda2652722 100644 --- a/crates/iota-indexer/src/store/mod.rs +++ b/crates/iota-indexer/src/store/mod.rs @@ -2,7 +2,7 @@ // Modifications Copyright (c) 2024 IOTA Stiftung // SPDX-License-Identifier: Apache-2.0 -pub(crate) use indexer_analytics_store::*; +pub(crate) use indexer_analytics_store::IndexerAnalyticalStore; pub(crate) use indexer_store::*; pub use pg_indexer_analytical_store::PgIndexerAnalyticalStore; pub use pg_indexer_store::PgIndexerStore; From a249473fb111560b5f4a00ee85921075e0e97157 Mon Sep 17 00:00:00 2001 From: samuel_rufi Date: Tue, 20 Aug 2024 15:32:01 +0200 Subject: [PATCH 10/15] refactor: Move processors --- crates/iota-indexer/src/handlers/mod.rs | 5 ----- crates/iota-indexer/src/indexer.rs | 2 ++ crates/iota-indexer/src/lib.rs | 1 + .../{handlers => processors}/address_metrics_processor.rs | 0 crates/iota-indexer/src/processors/mod.rs | 5 +++++ .../{handlers => processors}/move_call_metrics_processor.rs | 0 .../{handlers => processors}/network_metrics_processor.rs | 0 .../{handlers => processors}/objects_snapshot_processor.rs | 0 .../src/{handlers => processors}/processor_orchestrator.rs | 0 crates/iota-indexer/src/store/indexer_analytics_store.rs | 1 + crates/iota-indexer/src/test_utils.rs | 2 +- 11 files changed, 10 insertions(+), 6 deletions(-) rename crates/iota-indexer/src/{handlers => processors}/address_metrics_processor.rs (100%) create mode 100644 crates/iota-indexer/src/processors/mod.rs rename crates/iota-indexer/src/{handlers => processors}/move_call_metrics_processor.rs (100%) rename crates/iota-indexer/src/{handlers => processors}/network_metrics_processor.rs (100%) rename crates/iota-indexer/src/{handlers => processors}/objects_snapshot_processor.rs (100%) rename crates/iota-indexer/src/{handlers => processors}/processor_orchestrator.rs (100%) diff --git a/crates/iota-indexer/src/handlers/mod.rs b/crates/iota-indexer/src/handlers/mod.rs index 2f3fb5d553f..aef3b640bcb 100644 --- a/crates/iota-indexer/src/handlers/mod.rs +++ b/crates/iota-indexer/src/handlers/mod.rs @@ -12,13 +12,8 @@ use crate::{ }, }; -pub mod address_metrics_processor; pub mod checkpoint_handler; pub mod committer; -pub mod move_call_metrics_processor; -pub mod network_metrics_processor; -pub mod objects_snapshot_processor; -pub mod processor_orchestrator; pub mod tx_processor; #[derive(Debug)] diff --git a/crates/iota-indexer/src/indexer.rs b/crates/iota-indexer/src/indexer.rs index c7ecdf7122c..78b9ed7e623 100644 --- a/crates/iota-indexer/src/indexer.rs +++ b/crates/iota-indexer/src/indexer.rs @@ -15,6 +15,8 @@ use crate::{ framework::fetcher::CheckpointFetcher, handlers::{ checkpoint_handler::new_handlers, + }, + processors::{ objects_snapshot_processor::{ObjectsSnapshotProcessor, SnapshotLagConfig}, processor_orchestrator::ProcessorOrchestrator, }, diff --git a/crates/iota-indexer/src/lib.rs b/crates/iota-indexer/src/lib.rs index add2a3606ed..89f4bbdef28 100644 --- a/crates/iota-indexer/src/lib.rs +++ b/crates/iota-indexer/src/lib.rs @@ -36,6 +36,7 @@ pub mod metrics; pub mod models; pub mod schema; pub mod store; +pub mod processors; pub mod test_utils; pub mod types; diff --git a/crates/iota-indexer/src/handlers/address_metrics_processor.rs b/crates/iota-indexer/src/processors/address_metrics_processor.rs similarity index 100% rename from crates/iota-indexer/src/handlers/address_metrics_processor.rs rename to crates/iota-indexer/src/processors/address_metrics_processor.rs diff --git a/crates/iota-indexer/src/processors/mod.rs b/crates/iota-indexer/src/processors/mod.rs new file mode 100644 index 00000000000..14d7e654a10 --- /dev/null +++ b/crates/iota-indexer/src/processors/mod.rs @@ -0,0 +1,5 @@ +pub mod move_call_metrics_processor; +pub mod network_metrics_processor; +pub mod objects_snapshot_processor; +pub mod processor_orchestrator; +pub mod address_metrics_processor; diff --git a/crates/iota-indexer/src/handlers/move_call_metrics_processor.rs b/crates/iota-indexer/src/processors/move_call_metrics_processor.rs similarity index 100% rename from crates/iota-indexer/src/handlers/move_call_metrics_processor.rs rename to crates/iota-indexer/src/processors/move_call_metrics_processor.rs diff --git a/crates/iota-indexer/src/handlers/network_metrics_processor.rs b/crates/iota-indexer/src/processors/network_metrics_processor.rs similarity index 100% rename from crates/iota-indexer/src/handlers/network_metrics_processor.rs rename to crates/iota-indexer/src/processors/network_metrics_processor.rs diff --git a/crates/iota-indexer/src/handlers/objects_snapshot_processor.rs b/crates/iota-indexer/src/processors/objects_snapshot_processor.rs similarity index 100% rename from crates/iota-indexer/src/handlers/objects_snapshot_processor.rs rename to crates/iota-indexer/src/processors/objects_snapshot_processor.rs diff --git a/crates/iota-indexer/src/handlers/processor_orchestrator.rs b/crates/iota-indexer/src/processors/processor_orchestrator.rs similarity index 100% rename from crates/iota-indexer/src/handlers/processor_orchestrator.rs rename to crates/iota-indexer/src/processors/processor_orchestrator.rs diff --git a/crates/iota-indexer/src/store/indexer_analytics_store.rs b/crates/iota-indexer/src/store/indexer_analytics_store.rs index 1a3353e347c..3dbd5c657df 100644 --- a/crates/iota-indexer/src/store/indexer_analytics_store.rs +++ b/crates/iota-indexer/src/store/indexer_analytics_store.rs @@ -19,6 +19,7 @@ use crate::{ }; /// Provides methods to get and persist metrics. Utility methods for calculating metrics are also provided. +#[async_trait] pub trait IndexerAnalyticalStore { async fn get_latest_stored_transaction(&self) -> IndexerResult>; async fn get_latest_stored_checkpoint(&self) -> IndexerResult>; diff --git a/crates/iota-indexer/src/test_utils.rs b/crates/iota-indexer/src/test_utils.rs index 7f54fed35b7..6b49eedc1f2 100644 --- a/crates/iota-indexer/src/test_utils.rs +++ b/crates/iota-indexer/src/test_utils.rs @@ -13,7 +13,7 @@ use tracing::info; use crate::{ db::{new_pg_connection_pool_with_config, reset_database, PgConnectionPoolConfig}, errors::IndexerError, - handlers::objects_snapshot_processor::SnapshotLagConfig, + processors::objects_snapshot_processor::SnapshotLagConfig, indexer::Indexer, store::PgIndexerStore, IndexerConfig, IndexerMetrics, From 8b3174f4d5ae00cb85a67f223c7a80c0f6459ee3 Mon Sep 17 00:00:00 2001 From: samuel_rufi Date: Tue, 20 Aug 2024 15:37:40 +0200 Subject: [PATCH 11/15] refactor: Regenerate OpenRPC spec --- crates/iota-open-rpc/spec/openrpc.json | 833 +++++++++++++++++- .../src/generate_json_rpc_spec.rs | 5 +- 2 files changed, 798 insertions(+), 40 deletions(-) diff --git a/crates/iota-open-rpc/spec/openrpc.json b/crates/iota-open-rpc/spec/openrpc.json index d6c21b5750a..b05d4c64208 100644 --- a/crates/iota-open-rpc/spec/openrpc.json +++ b/crates/iota-open-rpc/spec/openrpc.json @@ -3259,6 +3259,32 @@ } ] }, + { + "name": "iotax_getAllEpochAddressMetrics", + "tags": [ + { + "name": "Extended API" + } + ], + "params": [ + { + "name": "descending_order", + "schema": { + "type": "boolean" + } + } + ], + "result": { + "name": "Vec", + "required": true, + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/AddressMetrics" + } + } + } + }, { "name": "iotax_getBalance", "tags": [ @@ -3316,6 +3342,32 @@ } ] }, + { + "name": "iotax_getCheckpointAddressMetrics", + "tags": [ + { + "name": "Extended API" + } + ], + "params": [ + { + "name": "checkpoint", + "required": true, + "schema": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + } + } + ], + "result": { + "name": "AddressMetrics", + "required": true, + "schema": { + "$ref": "#/components/schemas/AddressMetrics" + } + } + }, { "name": "iotax_getCoinMetadata", "tags": [ @@ -3528,6 +3580,23 @@ } ] }, + { + "name": "iotax_getCurrentEpoch", + "tags": [ + { + "name": "Extended API" + } + ], + "description": "Return current epoch info", + "params": [], + "result": { + "name": "EpochInfo", + "required": true, + "schema": { + "$ref": "#/components/schemas/EpochInfo" + } + } + }, { "name": "iotax_getDynamicFieldObject", "tags": [ @@ -3708,6 +3777,105 @@ } ] }, + { + "name": "iotax_getEpochMetrics", + "tags": [ + { + "name": "Extended API" + } + ], + "description": "Return a list of epoch metrics, which is a subset of epoch info", + "params": [ + { + "name": "cursor", + "description": "Optional paging cursor", + "schema": { + "$ref": "#/components/schemas/BigInt_for_uint64" + } + }, + { + "name": "limit", + "description": "Maximum number of items per page", + "schema": { + "type": "integer", + "format": "uint", + "minimum": 0.0 + } + }, + { + "name": "descending_order", + "description": "Flag to return results in descending order", + "schema": { + "type": "boolean" + } + } + ], + "result": { + "name": "EpochMetricsPage", + "required": true, + "schema": { + "$ref": "#/components/schemas/Page_for_EpochMetrics_and_BigInt_for_uint64" + } + } + }, + { + "name": "iotax_getEpochs", + "tags": [ + { + "name": "Extended API" + } + ], + "description": "Return a list of epoch info", + "params": [ + { + "name": "cursor", + "description": "Optional paging cursor", + "schema": { + "$ref": "#/components/schemas/BigInt_for_uint64" + } + }, + { + "name": "limit", + "description": "Maximum number of items per page", + "schema": { + "type": "integer", + "format": "uint", + "minimum": 0.0 + } + }, + { + "name": "descending_order", + "description": "Flag to return results in descending order", + "schema": { + "type": "boolean" + } + } + ], + "result": { + "name": "EpochPage", + "required": true, + "schema": { + "$ref": "#/components/schemas/Page_for_EpochInfo_and_BigInt_for_uint64" + } + } + }, + { + "name": "iotax_getLatestAddressMetrics", + "tags": [ + { + "name": "Extended API" + } + ], + "description": "Address related metrics", + "params": [], + "result": { + "name": "AddressMetrics", + "required": true, + "schema": { + "$ref": "#/components/schemas/AddressMetrics" + } + } + }, { "name": "iotax_getLatestIotaSystemState", "tags": [ @@ -3735,6 +3903,40 @@ } ] }, + { + "name": "iotax_getMoveCallMetrics", + "tags": [ + { + "name": "Extended API" + } + ], + "description": "Return move call metrics", + "params": [], + "result": { + "name": "MoveCallMetrics", + "required": true, + "schema": { + "$ref": "#/components/schemas/MoveCallMetrics" + } + } + }, + { + "name": "iotax_getNetworkMetrics", + "tags": [ + { + "name": "Extended API" + } + ], + "description": "Return Network metrics", + "params": [], + "result": { + "name": "NetworkMetrics", + "required": true, + "schema": { + "$ref": "#/components/schemas/NetworkMetrics" + } + } + }, { "name": "iotax_getOwnedObjects", "tags": [ @@ -4154,6 +4356,22 @@ } ] }, + { + "name": "iotax_getTotalTransactions", + "tags": [ + { + "name": "Extended API" + } + ], + "params": [], + "result": { + "name": "BigInt", + "required": true, + "schema": { + "$ref": "#/components/schemas/BigInt_for_uint64" + } + } + }, { "name": "iotax_getValidatorsApy", "tags": [ @@ -4337,6 +4555,48 @@ } ] }, + { + "name": "iotax_queryObjects", + "tags": [ + { + "name": "Extended API" + } + ], + "description": "Return the list of queried objects. Note that this is an enhanced full node only api.", + "params": [ + { + "name": "query", + "description": "The objects query criteria.", + "required": true, + "schema": { + "$ref": "#/components/schemas/ObjectResponseQuery" + } + }, + { + "name": "cursor", + "description": "An optional paging cursor. If provided, the query will start from the next item after the specified cursor. Default to start from the first item if not specified.", + "schema": { + "$ref": "#/components/schemas/CheckpointedObjectID" + } + }, + { + "name": "limit", + "description": "Max number of items returned per page, default to [QUERY_MAX_RESULT_LIMIT] if not specified.", + "schema": { + "type": "integer", + "format": "uint", + "minimum": 0.0 + } + } + ], + "result": { + "name": "QueryObjectsPage", + "required": true, + "schema": { + "$ref": "#/components/schemas/Page_for_IotaObjectResponse_and_CheckpointedObjectID" + } + } + }, { "name": "iotax_queryTransactionBlocks", "tags": [ @@ -5532,6 +5792,56 @@ ], "components": { "schemas": { + "AddressMetrics": { + "description": "Provides metrics about the addresses.", + "type": "object", + "required": [ + "checkpoint", + "cumulativeActiveAddresses", + "cumulativeAddresses", + "dailyActiveAddresses", + "epoch", + "timestampMs" + ], + "properties": { + "checkpoint": { + "description": "The checkpoint sequence number at which the metrics were computed.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "cumulativeActiveAddresses": { + "description": "The count of sender addresses.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "cumulativeAddresses": { + "description": "The count of sender and recipient addresses.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "dailyActiveAddresses": { + "description": "The count of daily unique sender addresses.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "epoch": { + "description": "The epoch to which the checkpoint is assigned.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "timestampMs": { + "description": "The checkpoint timestamp.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + } + } + }, "AuthorityPublicKeyBytes": { "description": "Defines the compressed version of the public key that we pass around in Iota", "allOf": [ @@ -5600,6 +5910,9 @@ "description": "Base64 encoding", "type": "string" }, + "BigInt_for_uint": { + "type": "string" + }, "BigInt_for_uint128": { "type": "string" }, @@ -5762,6 +6075,27 @@ } ] }, + "CheckpointedObjectID": { + "type": "object", + "required": [ + "objectId" + ], + "properties": { + "atCheckpoint": { + "anyOf": [ + { + "$ref": "#/components/schemas/BigInt_for_uint64" + }, + { + "type": "null" + } + ] + }, + "objectId": { + "$ref": "#/components/schemas/ObjectID" + } + } + }, "Claim": { "description": "A claim consists of value and index_mod_4.", "type": "object", @@ -6269,49 +6603,235 @@ "Ed25519IotaSignature": { "$ref": "#/components/schemas/Base64" }, - "EndOfEpochData": { + "EndOfEpochData": { + "type": "object", + "required": [ + "epochCommitments", + "epochSupplyChange", + "nextEpochCommittee", + "nextEpochProtocolVersion" + ], + "properties": { + "epochCommitments": { + "description": "Commitments to epoch specific state (e.g. live object set)", + "type": "array", + "items": { + "$ref": "#/components/schemas/CheckpointCommitment" + } + }, + "epochSupplyChange": { + "description": "The number of tokens that were minted (if positive) or burnt (if negative) in this epoch.", + "type": "integer", + "format": "int64" + }, + "nextEpochCommittee": { + "description": "next_epoch_committee is `Some` if and only if the current checkpoint is the last checkpoint of an epoch. Therefore next_epoch_committee can be used to pick the last checkpoint of an epoch, which is often useful to get epoch level summary stats like total gas cost of an epoch, or the total number of transactions from genesis to the end of an epoch. The committee is stored as a vector of validator pub key and stake pairs. The vector should be sorted based on the Committee data structure.", + "type": "array", + "items": { + "type": "array", + "items": [ + { + "$ref": "#/components/schemas/AuthorityPublicKeyBytes" + }, + { + "$ref": "#/components/schemas/BigInt_for_uint64" + } + ], + "maxItems": 2, + "minItems": 2 + } + }, + "nextEpochProtocolVersion": { + "description": "The protocol version that is in effect during the epoch that starts immediately after this checkpoint.", + "allOf": [ + { + "$ref": "#/components/schemas/ProtocolVersion" + } + ] + } + } + }, + "EndOfEpochInfo": { + "type": "object", + "required": [ + "burntTokensAmount", + "epochEndTimestamp", + "lastCheckpointId", + "mintedTokensAmount", + "protocolVersion", + "referenceGasPrice", + "storageCharge", + "storageFundBalance", + "storageRebate", + "totalGasFees", + "totalStake", + "totalStakeRewardsDistributed" + ], + "properties": { + "burntTokensAmount": { + "$ref": "#/components/schemas/BigInt_for_uint64" + }, + "epochEndTimestamp": { + "$ref": "#/components/schemas/BigInt_for_uint64" + }, + "lastCheckpointId": { + "$ref": "#/components/schemas/BigInt_for_uint64" + }, + "mintedTokensAmount": { + "$ref": "#/components/schemas/BigInt_for_uint64" + }, + "protocolVersion": { + "description": "existing fields from `SystemEpochInfoEvent` (without epoch)", + "allOf": [ + { + "$ref": "#/components/schemas/BigInt_for_uint64" + } + ] + }, + "referenceGasPrice": { + "$ref": "#/components/schemas/BigInt_for_uint64" + }, + "storageCharge": { + "$ref": "#/components/schemas/BigInt_for_uint64" + }, + "storageFundBalance": { + "$ref": "#/components/schemas/BigInt_for_uint64" + }, + "storageRebate": { + "$ref": "#/components/schemas/BigInt_for_uint64" + }, + "totalGasFees": { + "$ref": "#/components/schemas/BigInt_for_uint64" + }, + "totalStake": { + "$ref": "#/components/schemas/BigInt_for_uint64" + }, + "totalStakeRewardsDistributed": { + "$ref": "#/components/schemas/BigInt_for_uint64" + } + } + }, + "EpochInfo": { + "type": "object", + "required": [ + "epoch", + "epochStartTimestamp", + "epochTotalTransactions", + "firstCheckpointId", + "validators" + ], + "properties": { + "endOfEpochInfo": { + "description": "The end of epoch information.", + "anyOf": [ + { + "$ref": "#/components/schemas/EndOfEpochInfo" + }, + { + "type": "null" + } + ] + }, + "epoch": { + "description": "Epoch number", + "allOf": [ + { + "$ref": "#/components/schemas/BigInt_for_uint64" + } + ] + }, + "epochStartTimestamp": { + "description": "The timestamp when the epoch started.", + "allOf": [ + { + "$ref": "#/components/schemas/BigInt_for_uint64" + } + ] + }, + "epochTotalTransactions": { + "description": "Count of tx in epoch", + "allOf": [ + { + "$ref": "#/components/schemas/BigInt_for_uint64" + } + ] + }, + "firstCheckpointId": { + "description": "First, last checkpoint sequence numbers", + "allOf": [ + { + "$ref": "#/components/schemas/BigInt_for_uint64" + } + ] + }, + "referenceGasPrice": { + "description": "The reference gas price for the given epoch.", + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "validators": { + "description": "List of validators included in epoch", + "type": "array", + "items": { + "$ref": "#/components/schemas/IotaValidatorSummary" + } + } + } + }, + "EpochMetrics": { + "description": "A light-weight version of `EpochInfo` for faster loading", "type": "object", "required": [ - "epochCommitments", - "epochSupplyChange", - "nextEpochCommittee", - "nextEpochProtocolVersion" + "epoch", + "epochStartTimestamp", + "epochTotalTransactions", + "firstCheckpointId" ], "properties": { - "epochCommitments": { - "description": "Commitments to epoch specific state (e.g. live object set)", - "type": "array", - "items": { - "$ref": "#/components/schemas/CheckpointCommitment" - } + "endOfEpochInfo": { + "description": "The end of epoch information.", + "anyOf": [ + { + "$ref": "#/components/schemas/EndOfEpochInfo" + }, + { + "type": "null" + } + ] }, - "epochSupplyChange": { - "description": "The number of tokens that were minted (if positive) or burnt (if negative) in this epoch.", - "type": "integer", - "format": "int64" + "epoch": { + "description": "The current epoch ID.", + "allOf": [ + { + "$ref": "#/components/schemas/BigInt_for_uint64" + } + ] }, - "nextEpochCommittee": { - "description": "next_epoch_committee is `Some` if and only if the current checkpoint is the last checkpoint of an epoch. Therefore next_epoch_committee can be used to pick the last checkpoint of an epoch, which is often useful to get epoch level summary stats like total gas cost of an epoch, or the total number of transactions from genesis to the end of an epoch. The committee is stored as a vector of validator pub key and stake pairs. The vector should be sorted based on the Committee data structure.", - "type": "array", - "items": { - "type": "array", - "items": [ - { - "$ref": "#/components/schemas/AuthorityPublicKeyBytes" - }, - { - "$ref": "#/components/schemas/BigInt_for_uint64" - } - ], - "maxItems": 2, - "minItems": 2 - } + "epochStartTimestamp": { + "description": "The timestamp when the epoch started.", + "allOf": [ + { + "$ref": "#/components/schemas/BigInt_for_uint64" + } + ] }, - "nextEpochProtocolVersion": { - "description": "The protocol version that is in effect during the epoch that starts immediately after this checkpoint.", + "epochTotalTransactions": { + "description": "The total number of transactions in the epoch.", "allOf": [ { - "$ref": "#/components/schemas/ProtocolVersion" + "$ref": "#/components/schemas/BigInt_for_uint64" + } + ] + }, + "firstCheckpointId": { + "description": "The first checkpoint ID of the epoch.", + "allOf": [ + { + "$ref": "#/components/schemas/BigInt_for_uint64" } ] } @@ -8633,6 +9153,67 @@ } } }, + "MoveCallMetrics": { + "type": "object", + "required": [ + "rank30Days", + "rank3Days", + "rank7Days" + ], + "properties": { + "rank30Days": { + "description": "The count of calls of each function in the last 30 days.", + "type": "array", + "items": { + "type": "array", + "items": [ + { + "$ref": "#/components/schemas/MoveFunctionName" + }, + { + "$ref": "#/components/schemas/BigInt_for_uint" + } + ], + "maxItems": 2, + "minItems": 2 + } + }, + "rank3Days": { + "description": "The count of calls of each function in the last 3 days.", + "type": "array", + "items": { + "type": "array", + "items": [ + { + "$ref": "#/components/schemas/MoveFunctionName" + }, + { + "$ref": "#/components/schemas/BigInt_for_uint" + } + ], + "maxItems": 2, + "minItems": 2 + } + }, + "rank7Days": { + "description": "The count of calls of each function in the last 7 days.", + "type": "array", + "items": { + "type": "array", + "items": [ + { + "$ref": "#/components/schemas/MoveFunctionName" + }, + { + "$ref": "#/components/schemas/BigInt_for_uint" + } + ], + "maxItems": 2, + "minItems": 2 + } + } + } + }, "MoveCallParams": { "type": "object", "required": [ @@ -8688,6 +9269,33 @@ } ] }, + "MoveFunctionName": { + "description": "Identifies a Move function.", + "type": "object", + "required": [ + "function", + "module", + "package" + ], + "properties": { + "function": { + "description": "The function name.", + "type": "string" + }, + "module": { + "description": "The module name to which the function belongs.", + "type": "string" + }, + "package": { + "description": "The package ID to which the function belongs.", + "allOf": [ + { + "$ref": "#/components/schemas/ObjectID" + } + ] + } + } + }, "MoveStruct": { "anyOf": [ { @@ -8906,6 +9514,70 @@ } } }, + "NetworkMetrics": { + "type": "object", + "required": [ + "currentCheckpoint", + "currentEpoch", + "currentTps", + "totalAddresses", + "totalObjects", + "totalPackages", + "tps30Days" + ], + "properties": { + "currentCheckpoint": { + "description": "Current checkpoint number", + "allOf": [ + { + "$ref": "#/components/schemas/BigInt_for_uint64" + } + ] + }, + "currentEpoch": { + "description": "Current epoch number", + "allOf": [ + { + "$ref": "#/components/schemas/BigInt_for_uint64" + } + ] + }, + "currentTps": { + "description": "Current TPS - Transaction Blocks per Second.", + "type": "number", + "format": "double" + }, + "totalAddresses": { + "description": "Total number of addresses seen in the network", + "allOf": [ + { + "$ref": "#/components/schemas/BigInt_for_uint64" + } + ] + }, + "totalObjects": { + "description": "Total number of live objects in the network", + "allOf": [ + { + "$ref": "#/components/schemas/BigInt_for_uint64" + } + ] + }, + "totalPackages": { + "description": "Total number of packages published in the network", + "allOf": [ + { + "$ref": "#/components/schemas/BigInt_for_uint64" + } + ] + }, + "tps30Days": { + "description": "Peak TPS in the past 30 days", + "type": "number", + "format": "double" + } + } + }, "ObjectChange": { "description": "ObjectChange are derived from the object mutations in the TransactionEffect to provide richer object information.", "oneOf": [ @@ -9745,6 +10417,64 @@ } } }, + "Page_for_EpochInfo_and_BigInt_for_uint64": { + "description": "`next_cursor` points to the last item in the page; Reading with `next_cursor` will start from the next item after `next_cursor` if `next_cursor` is `Some`, otherwise it will start from the first item.", + "type": "object", + "required": [ + "data", + "hasNextPage" + ], + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/components/schemas/EpochInfo" + } + }, + "hasNextPage": { + "type": "boolean" + }, + "nextCursor": { + "anyOf": [ + { + "$ref": "#/components/schemas/BigInt_for_uint64" + }, + { + "type": "null" + } + ] + } + } + }, + "Page_for_EpochMetrics_and_BigInt_for_uint64": { + "description": "`next_cursor` points to the last item in the page; Reading with `next_cursor` will start from the next item after `next_cursor` if `next_cursor` is `Some`, otherwise it will start from the first item.", + "type": "object", + "required": [ + "data", + "hasNextPage" + ], + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/components/schemas/EpochMetrics" + } + }, + "hasNextPage": { + "type": "boolean" + }, + "nextCursor": { + "anyOf": [ + { + "$ref": "#/components/schemas/BigInt_for_uint64" + }, + { + "type": "null" + } + ] + } + } + }, "Page_for_Event_and_EventID": { "description": "`next_cursor` points to the last item in the page; Reading with `next_cursor` will start from the next item after `next_cursor` if `next_cursor` is `Some`, otherwise it will start from the first item.", "type": "object", @@ -9774,6 +10504,35 @@ } } }, + "Page_for_IotaObjectResponse_and_CheckpointedObjectID": { + "description": "`next_cursor` points to the last item in the page; Reading with `next_cursor` will start from the next item after `next_cursor` if `next_cursor` is `Some`, otherwise it will start from the first item.", + "type": "object", + "required": [ + "data", + "hasNextPage" + ], + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/components/schemas/IotaObjectResponse" + } + }, + "hasNextPage": { + "type": "boolean" + }, + "nextCursor": { + "anyOf": [ + { + "$ref": "#/components/schemas/CheckpointedObjectID" + }, + { + "type": "null" + } + ] + } + } + }, "Page_for_IotaObjectResponse_and_ObjectID": { "description": "`next_cursor` points to the last item in the page; Reading with `next_cursor` will start from the next item after `next_cursor` if `next_cursor` is `Some`, otherwise it will start from the first item.", "type": "object", @@ -10120,13 +10879,13 @@ "$ref": "#/components/schemas/Base64" }, "SequenceNumber": { + "$ref": "#/components/schemas/BigInt_for_uint64" + }, + "SequenceNumber2": { "type": "integer", "format": "uint64", "minimum": 0.0 }, - "SequenceNumber2": { - "$ref": "#/components/schemas/BigInt_for_uint64" - }, "Signature": { "oneOf": [ { diff --git a/crates/iota-open-rpc/src/generate_json_rpc_spec.rs b/crates/iota-open-rpc/src/generate_json_rpc_spec.rs index e8a5f245308..30624f4fe15 100644 --- a/crates/iota-open-rpc/src/generate_json_rpc_spec.rs +++ b/crates/iota-open-rpc/src/generate_json_rpc_spec.rs @@ -13,7 +13,7 @@ use iota_json_rpc::{ transaction_builder_api::TransactionBuilderApi, transaction_execution_api::TransactionExecutionApi, IotaRpcModule, }; -use iota_json_rpc_api::{IndexerApiOpenRpc, MoveUtilsOpenRpc}; +use iota_json_rpc_api::{ExtendedApiOpenRpc, IndexerApiOpenRpc, MoveUtilsOpenRpc}; use pretty_assertions::assert_str_eq; use crate::examples::RpcExampleProvider; @@ -53,8 +53,7 @@ async fn main() { open_rpc.add_module(TransactionExecutionApi::rpc_doc_module()); open_rpc.add_module(TransactionBuilderApi::rpc_doc_module()); open_rpc.add_module(GovernanceReadApi::rpc_doc_module()); - // temporarily remove api ref content for indexer methods - // open_rpc.add_module(ExtendedApiOpenRpc::module_doc()); + open_rpc.add_module(ExtendedApiOpenRpc::module_doc()); open_rpc.add_module(MoveUtilsOpenRpc::module_doc()); open_rpc.add_examples(RpcExampleProvider::new().examples()); From bd9df456ea1f285c1e187d91024cf4f5fdf0c8e3 Mon Sep 17 00:00:00 2001 From: samuel_rufi Date: Tue, 20 Aug 2024 15:42:46 +0200 Subject: [PATCH 12/15] refactor: Add license --- crates/iota-indexer/src/indexer.rs | 8 +++----- crates/iota-indexer/src/lib.rs | 2 +- crates/iota-indexer/src/processors/mod.rs | 6 +++++- crates/iota-indexer/src/store/indexer_analytics_store.rs | 3 ++- .../iota-indexer/src/store/pg_indexer_analytical_store.rs | 3 ++- crates/iota-indexer/src/test_utils.rs | 2 +- 6 files changed, 14 insertions(+), 10 deletions(-) diff --git a/crates/iota-indexer/src/indexer.rs b/crates/iota-indexer/src/indexer.rs index 78b9ed7e623..f74c2a36714 100644 --- a/crates/iota-indexer/src/indexer.rs +++ b/crates/iota-indexer/src/indexer.rs @@ -13,15 +13,13 @@ use crate::{ build_json_rpc_server, errors::IndexerError, framework::fetcher::CheckpointFetcher, - handlers::{ - checkpoint_handler::new_handlers, - }, + handlers::checkpoint_handler::new_handlers, + indexer_reader::IndexerReader, + metrics::IndexerMetrics, processors::{ objects_snapshot_processor::{ObjectsSnapshotProcessor, SnapshotLagConfig}, processor_orchestrator::ProcessorOrchestrator, }, - indexer_reader::IndexerReader, - metrics::IndexerMetrics, store::{IndexerStore, PgIndexerAnalyticalStore}, IndexerConfig, }; diff --git a/crates/iota-indexer/src/lib.rs b/crates/iota-indexer/src/lib.rs index 89f4bbdef28..edd882c6db5 100644 --- a/crates/iota-indexer/src/lib.rs +++ b/crates/iota-indexer/src/lib.rs @@ -34,9 +34,9 @@ pub mod indexer; pub mod indexer_reader; pub mod metrics; pub mod models; +pub mod processors; pub mod schema; pub mod store; -pub mod processors; pub mod test_utils; pub mod types; diff --git a/crates/iota-indexer/src/processors/mod.rs b/crates/iota-indexer/src/processors/mod.rs index 14d7e654a10..a43484977b1 100644 --- a/crates/iota-indexer/src/processors/mod.rs +++ b/crates/iota-indexer/src/processors/mod.rs @@ -1,5 +1,9 @@ +// Copyright (c) Mysten Labs, Inc. +// Modifications Copyright (c) 2024 IOTA Stiftung +// SPDX-License-Identifier: Apache-2.0 + +pub mod address_metrics_processor; pub mod move_call_metrics_processor; pub mod network_metrics_processor; pub mod objects_snapshot_processor; pub mod processor_orchestrator; -pub mod address_metrics_processor; diff --git a/crates/iota-indexer/src/store/indexer_analytics_store.rs b/crates/iota-indexer/src/store/indexer_analytics_store.rs index 3dbd5c657df..d32116b540e 100644 --- a/crates/iota-indexer/src/store/indexer_analytics_store.rs +++ b/crates/iota-indexer/src/store/indexer_analytics_store.rs @@ -18,7 +18,8 @@ use crate::{ types::IndexerResult, }; -/// Provides methods to get and persist metrics. Utility methods for calculating metrics are also provided. +/// Provides methods to get and persist metrics. Utility methods for calculating +/// metrics are also provided. #[async_trait] pub trait IndexerAnalyticalStore { async fn get_latest_stored_transaction(&self) -> IndexerResult>; diff --git a/crates/iota-indexer/src/store/pg_indexer_analytical_store.rs b/crates/iota-indexer/src/store/pg_indexer_analytical_store.rs index 2bd104d8c52..496ff491131 100644 --- a/crates/iota-indexer/src/store/pg_indexer_analytical_store.rs +++ b/crates/iota-indexer/src/store/pg_indexer_analytical_store.rs @@ -37,7 +37,8 @@ use crate::{ types::IndexerResult, }; -/// The store for the indexer analytical data. Represents a Postgres implementation of the `IndexerAnalyticalStore` trait. +/// The store for the indexer analytical data. Represents a Postgres +/// implementation of the `IndexerAnalyticalStore` trait. #[derive(Clone)] pub struct PgIndexerAnalyticalStore { blocking_cp: PgConnectionPool, diff --git a/crates/iota-indexer/src/test_utils.rs b/crates/iota-indexer/src/test_utils.rs index 6b49eedc1f2..beaf18345cb 100644 --- a/crates/iota-indexer/src/test_utils.rs +++ b/crates/iota-indexer/src/test_utils.rs @@ -13,8 +13,8 @@ use tracing::info; use crate::{ db::{new_pg_connection_pool_with_config, reset_database, PgConnectionPoolConfig}, errors::IndexerError, - processors::objects_snapshot_processor::SnapshotLagConfig, indexer::Indexer, + processors::objects_snapshot_processor::SnapshotLagConfig, store::PgIndexerStore, IndexerConfig, IndexerMetrics, }; From ca24fe81831492ab21bdcddcab5d70bf1973c1d8 Mon Sep 17 00:00:00 2001 From: samuel_rufi Date: Tue, 20 Aug 2024 15:48:27 +0200 Subject: [PATCH 13/15] refactor: Fix import --- crates/iota-graphql-rpc/src/test_infra/cluster.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/iota-graphql-rpc/src/test_infra/cluster.rs b/crates/iota-graphql-rpc/src/test_infra/cluster.rs index 8ad0ffa63aa..f23e774a75f 100644 --- a/crates/iota-graphql-rpc/src/test_infra/cluster.rs +++ b/crates/iota-graphql-rpc/src/test_infra/cluster.rs @@ -5,7 +5,7 @@ use std::{net::SocketAddr, sync::Arc, time::Duration}; use iota_graphql_rpc_client::simple_client::SimpleClient; -pub use iota_indexer::handlers::objects_snapshot_processor::SnapshotLagConfig; +pub use iota_indexer::processors::objects_snapshot_processor::SnapshotLagConfig; use iota_indexer::{ errors::IndexerError, store::{indexer_store::IndexerStore, PgIndexerStore}, From 7a0bb1fe2220857c04a4179ea0bca516b4eb52d6 Mon Sep 17 00:00:00 2001 From: Samuel Rufinatscha Date: Tue, 20 Aug 2024 17:27:29 +0200 Subject: [PATCH 14/15] refactor: Remove outdated comment --- crates/iota-indexer/src/processors/objects_snapshot_processor.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/iota-indexer/src/processors/objects_snapshot_processor.rs b/crates/iota-indexer/src/processors/objects_snapshot_processor.rs index 7da7903c87d..4f4758a2d29 100644 --- a/crates/iota-indexer/src/processors/objects_snapshot_processor.rs +++ b/crates/iota-indexer/src/processors/objects_snapshot_processor.rs @@ -57,7 +57,6 @@ impl Default for SnapshotLagConfig { } } -// TODO: "handler" impl ObjectsSnapshotProcessor where S: IndexerStore + Clone + Sync + Send + 'static, From 2c3959f7b0f6d4b03179bffab49d687307c368a0 Mon Sep 17 00:00:00 2001 From: Samuel Rufinatscha Date: Tue, 20 Aug 2024 17:33:33 +0200 Subject: [PATCH 15/15] refactor: Add comments to clarify `address` vs `active_address` meaning --- crates/iota-indexer/src/metrics.rs | 4 ++-- crates/iota-indexer/src/models/address_metrics.rs | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/iota-indexer/src/metrics.rs b/crates/iota-indexer/src/metrics.rs index 4385b97d5ce..65c3f5b0ba9 100644 --- a/crates/iota-indexer/src/metrics.rs +++ b/crates/iota-indexer/src/metrics.rs @@ -91,11 +91,11 @@ pub struct IndexerMetrics { pub latest_tx_checkpoint_sequence_number: IntGauge, pub latest_indexer_object_checkpoint_sequence_number: IntGauge, pub latest_object_snapshot_sequence_number: IntGauge, - // analytical + // Analytical pub latest_move_call_metrics_tx_seq: IntGauge, pub latest_address_metrics_tx_seq: IntGauge, pub latest_network_metrics_cp_seq: IntGauge, - // checkpoint E2E latency is: + // Checkpoint E2E latency is: // fullnode_download_latency + checkpoint_index_latency + db_commit_latency pub checkpoint_download_bytes_size: IntGauge, pub fullnode_checkpoint_data_download_latency: Histogram, diff --git a/crates/iota-indexer/src/models/address_metrics.rs b/crates/iota-indexer/src/models/address_metrics.rs index a6e78256f2e..2e7406614c7 100644 --- a/crates/iota-indexer/src/models/address_metrics.rs +++ b/crates/iota-indexer/src/models/address_metrics.rs @@ -9,6 +9,7 @@ use iota_json_rpc_types::AddressMetrics; use crate::schema::{active_addresses, address_metrics, addresses}; +/// Represents a sender or receiver address. #[derive(Clone, Debug, Queryable, Insertable)] #[diesel(table_name = addresses)] pub struct StoredAddress { @@ -19,6 +20,7 @@ pub struct StoredAddress { pub last_appearance_time: i64, } +/// Represents a sender address. #[derive(Clone, Debug, Queryable, Insertable)] #[diesel(table_name = active_addresses)] pub struct StoredActiveAddress {