From 2be44673939c8d15e19fa6e5c18a02f4ed8f257c Mon Sep 17 00:00:00 2001 From: Kirill Ivanov <8144358+bragov4ik@users.noreply.github.com> Date: Wed, 11 Dec 2024 17:36:41 +0900 Subject: [PATCH] feat(stats): The Migration of The Backend charts: Total Blocks and Dynamic query dispatch (#1137) #1080 It is a sequel to #1120 and the third installment in the Migrate Backend Charts franchise. In this standalone sequel, we introduce reimagination of TotalBlocks counter and dynamic dispatch for chart query behaviour. TotalBlocks counter uses a custom logic with fallback to postgres catalog for preliminary block number estimation (to display when SQL query is not yet calculated), so we can't keep retrieving the data directly from DB. We need to allow dynamic query logic that is configurable for each chart. While adding usage of dynamic logic for chart query, regular date range was replaced by universal range, because chart requests can include or omit any of the bounds. --- stats/Cargo.lock | 15 +- stats/Cargo.toml | 1 + stats/config/update_groups.json | 2 +- stats/stats-server/src/read_service.rs | 318 ++++++----- stats/stats-server/src/runtime_setup.rs | 120 +++-- stats/stats-server/src/serializers.rs | 22 - stats/stats-server/src/server.rs | 17 +- stats/stats-server/src/update_service.rs | 26 +- .../stats-server/tests/it/indexing_status.rs | 6 +- stats/stats/Cargo.toml | 2 + stats/stats/src/charts/chart.rs | 46 +- .../src/charts/counters/average_block_time.rs | 28 +- .../src/charts/counters/last_new_contracts.rs | 6 +- .../counters/last_new_verified_contracts.rs | 5 +- stats/stats/src/charts/counters/mock.rs | 10 +- .../src/charts/counters/total_accounts.rs | 6 +- .../stats/src/charts/counters/total_blocks.rs | 124 +++-- .../src/charts/counters/total_contracts.rs | 15 +- .../counters/total_native_coin_holders.rs | 5 +- .../counters/total_verified_contracts.rs | 5 +- .../src/charts/counters/yesterday_txns.rs | 15 +- stats/stats/src/charts/db_interaction/read.rs | 194 ++++--- .../stats/src/charts/db_interaction/write.rs | 2 +- .../stats/src/charts/lines/accounts_growth.rs | 11 +- .../stats/src/charts/lines/active_accounts.rs | 12 +- .../src/charts/lines/average_block_rewards.rs | 22 +- .../src/charts/lines/average_block_size.rs | 26 +- .../src/charts/lines/average_gas_limit.rs | 22 +- .../src/charts/lines/average_gas_price.rs | 22 +- .../stats/src/charts/lines/average_txn_fee.rs | 22 +- .../src/charts/lines/contracts_growth.rs | 10 +- .../stats/src/charts/lines/gas_used_growth.rs | 27 +- stats/stats/src/charts/lines/mock.rs | 32 +- .../lines/native_coin_holders_growth.rs | 77 +-- .../src/charts/lines/native_coin_supply.rs | 23 +- stats/stats/src/charts/lines/new_accounts.rs | 38 +- .../src/charts/lines/new_block_rewards.rs | 41 +- stats/stats/src/charts/lines/new_blocks.rs | 129 ++--- stats/stats/src/charts/lines/new_contracts.rs | 18 +- .../charts/lines/new_native_coin_holders.rs | 6 +- .../charts/lines/new_native_coin_transfers.rs | 24 +- stats/stats/src/charts/lines/new_txns.rs | 18 +- .../charts/lines/new_verified_contracts.rs | 23 +- stats/stats/src/charts/lines/txns_fee.rs | 17 +- stats/stats/src/charts/lines/txns_growth.rs | 10 +- .../src/charts/lines/txns_success_rate.rs | 22 +- .../charts/lines/verified_contracts_growth.rs | 10 +- stats/stats/src/charts/mod.rs | 5 +- stats/stats/src/charts/query_dispatch.rs | 192 +++++++ stats/stats/src/charts/types/timespans/day.rs | 14 +- .../stats/src/charts/types/timespans/month.rs | 68 ++- .../stats/src/charts/types/timespans/week.rs | 30 +- .../stats/src/charts/types/timespans/year.rs | 91 +++- stats/stats/src/charts/types/traits.rs | 18 +- stats/stats/src/data_processing.rs | 8 +- .../data_source/kinds/auxiliary/cumulative.rs | 14 +- .../kinds/data_manipulation/delta.rs | 33 +- .../data_manipulation/filter_deducible.rs | 39 +- .../kinds/data_manipulation/last_point.rs | 18 +- .../kinds/data_manipulation/map/mod.rs | 19 +- .../kinds/data_manipulation/map/parse.rs | 14 +- .../data_manipulation/map/strip_extension.rs | 36 ++ .../kinds/data_manipulation/map/to_string.rs | 6 +- .../data_manipulation/resolutions/average.rs | 68 ++- .../resolutions/last_value.rs | 45 +- .../data_manipulation/resolutions/mod.rs | 73 ++- .../data_manipulation/resolutions/sum.rs | 49 +- .../kinds/data_manipulation/sum_point.rs | 19 +- .../src/data_source/kinds/local_db/mod.rs | 60 ++- .../kinds/local_db/parameter_traits.rs | 19 +- .../kinds/local_db/parameters/mod.rs | 4 +- .../kinds/local_db/parameters/query.rs | 180 +++++-- .../parameters/update/batching/mod.rs | 68 ++- .../update/batching/parameter_traits.rs | 4 +- .../update/batching/parameters/cumulative.rs | 6 +- .../update/batching/parameters/mock.rs | 4 +- .../update/batching/parameters/mod.rs | 6 +- .../kinds/local_db/parameters/update/point.rs | 11 +- .../src/data_source/kinds/remote_db/mod.rs | 14 +- .../data_source/kinds/remote_db/query/all.rs | 32 +- .../data_source/kinds/remote_db/query/each.rs | 19 +- .../data_source/kinds/remote_db/query/one.rs | 21 +- stats/stats/src/data_source/source.rs | 35 +- stats/stats/src/data_source/tests.rs | 36 +- stats/stats/src/data_source/types.rs | 10 +- stats/stats/src/lib.rs | 13 +- stats/stats/src/range.rs | 506 ++++++++++++++++++ stats/stats/src/tests/init_db.rs | 10 + stats/stats/src/tests/simple_test.rs | 193 +++---- stats/stats/src/update_group.rs | 73 ++- stats/stats/src/utils.rs | 66 ++- 91 files changed, 2651 insertions(+), 1250 deletions(-) create mode 100644 stats/stats/src/charts/query_dispatch.rs create mode 100644 stats/stats/src/data_source/kinds/data_manipulation/map/strip_extension.rs create mode 100644 stats/stats/src/range.rs diff --git a/stats/Cargo.lock b/stats/Cargo.lock index c08bb95ff..0b63993cf 100644 --- a/stats/Cargo.lock +++ b/stats/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "actix-codec" @@ -4731,10 +4731,12 @@ dependencies = [ "rust_decimal", "rust_decimal_macros", "sea-orm", + "stats-proto", "thiserror", "tokio", "tracing", "tracing-subscriber", + "trait-variant", "tynm", "url", "wiremock", @@ -5330,6 +5332,17 @@ dependencies = [ "tracing-serde", ] +[[package]] +name = "trait-variant" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70977707304198400eb4835a78f6a9f928bf41bba420deb8fdb175cd965d77a7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.79", +] + [[package]] name = "try-lock" version = "0.2.5" diff --git a/stats/Cargo.toml b/stats/Cargo.toml index 0a9975403..82cf74972 100644 --- a/stats/Cargo.toml +++ b/stats/Cargo.toml @@ -13,6 +13,7 @@ members = [ blockscout-client = { git = "https://github.com/blockscout/blockscout-rs/", rev = "506b821" } blockscout-service-launcher = { version = "0.13.1" } rstest = "0.23.0" +trait-variant = "0.1.2" wiremock = "0.6.2" # todo: update version after https://github.com/chronotope/chrono/pull/1600 diff --git a/stats/config/update_groups.json b/stats/config/update_groups.json index 870e8ba57..c07dab967 100644 --- a/stats/config/update_groups.json +++ b/stats/config/update_groups.json @@ -4,7 +4,7 @@ "average_block_time_group": "0 */10 * * * * *", "completed_txns_group": "0 5 */3 * * * *", "total_addresses_group": "0 0 */3 * * * *", - "total_blocks_group": "0 0 */3 * * * *", + "total_blocks_group": "0 0 */2 * * * *", "total_tokens_group": "0 0 18 * * * *", "yesterday_txns_group": "0 8 0 * * * *", "active_recurring_accounts_daily_recurrence_60_days_group": "0 0 2 * * * *", diff --git a/stats/stats-server/src/read_service.rs b/stats/stats-server/src/read_service.rs index 7f8dc8fbe..9ac964344 100644 --- a/stats/stats-server/src/read_service.rs +++ b/stats/stats-server/src/read_service.rs @@ -1,41 +1,51 @@ -use std::{clone::Clone, cmp::Ord, collections::BTreeMap, fmt::Debug, str::FromStr, sync::Arc}; +use std::{clone::Clone, collections::BTreeMap, fmt::Debug, str::FromStr, sync::Arc}; use crate::{ - config::{layout::sorted_items_according_to_layout, types}, + config::{ + layout::sorted_items_according_to_layout, + types::{self, EnabledChartSettings}, + }, runtime_setup::{EnabledChartEntry, RuntimeSetup}, - serializers::serialize_line_points, settings::LimitsSettings, }; use async_trait::async_trait; -use chrono::{NaiveDate, Utc}; +use chrono::{DateTime, NaiveDate, Utc}; +use futures::{stream::FuturesOrdered, StreamExt}; use proto_v1::stats_service_server::StatsService; -use sea_orm::{DatabaseConnection, DbErr}; +use sea_orm::DbErr; use stats::{ - entity::sea_orm_active_enums::ChartType, - types::{ - timespans::{Month, Week, Year}, - Timespan, - }, - ApproxUnsignedDiff, MissingDatePolicy, ReadError, RequestedPointsLimit, ResolutionKind, + data_source::{types::BlockscoutMigrations, UpdateContext, UpdateParameters}, + query_dispatch::{CounterHandle, LineHandle, QuerySerializedDyn}, + range::UniversalRange, + types::Timespan, + utils::{day_start, MarkedDbConnection}, + ChartError, RequestedPointsLimit, ResolutionKind, }; -use stats_proto::blockscout::stats::v1::{self as proto_v1, Point}; +use stats_proto::blockscout::stats::v1 as proto_v1; use tonic::{Request, Response, Status}; #[derive(Clone)] pub struct ReadService { - db: Arc, + db: MarkedDbConnection, + blockscout: MarkedDbConnection, charts: Arc, limits: ReadLimits, } impl ReadService { pub async fn new( - db: Arc, + db: MarkedDbConnection, + blockscout: MarkedDbConnection, charts: Arc, limits: ReadLimits, ) -> Result { - Ok(Self { db, charts, limits }) + Ok(Self { + db, + blockscout, + charts, + limits, + }) } } @@ -53,10 +63,10 @@ impl From for ReadLimits { } } -fn map_read_error(err: ReadError) -> Status { +fn map_update_error(err: ChartError) -> Status { match &err { - ReadError::ChartNotFound(_) => Status::not_found(err.to_string()), - ReadError::IntervalTooLarge(_) => Status::invalid_argument(err.to_string()), + ChartError::ChartNotFound(_) => Status::not_found(err.to_string()), + ChartError::IntervalTooLarge { limit: _ } => Status::invalid_argument(err.to_string()), _ => { tracing::error!(err = ?err, "internal read error"); Status::internal(err.to_string()) @@ -69,11 +79,11 @@ fn map_read_error(err: ReadError) -> Status { /// Returns `None` if info were not found for some chart. fn add_chart_info_to_layout( layout: Vec, - chart_info: BTreeMap, + chart_info: &BTreeMap, ) -> Vec { layout .into_iter() - .map(|cat| cat.intersect_info(&chart_info)) + .map(|cat| cat.intersect_info(chart_info)) .collect() } @@ -86,95 +96,64 @@ fn convert_resolution(input: proto_v1::Resolution) -> ResolutionKind { } } -async fn get_serialized_line_chart_data( - db: &DatabaseConnection, - chart_name: String, - from: Option, - to: Option, - points_limit: Option, - policy: MissingDatePolicy, - mark_approx: u64, -) -> Result, ReadError> -where - Resolution: Timespan + ApproxUnsignedDiff + Clone + Ord + Debug, -{ - let from = from.map(|f| Resolution::from_date(f)); - let to = to.map(|t| Resolution::from_date(t)); - let data = stats::get_line_chart_data::( - db, - &chart_name, - from, - to, - points_limit, - policy, - true, - mark_approx, - ) - .await?; - Ok(serialize_line_points(data)) -} - -/// enum dispatch for `get_serialized_line_chart_data` -#[allow(clippy::too_many_arguments)] -async fn get_serialized_line_chart_data_resolution_dispatch( - db: &DatabaseConnection, - chart_name: String, - resolution: ResolutionKind, - from: Option, - to: Option, - points_limit: Option, - policy: MissingDatePolicy, - mark_approx: u64, -) -> Result, ReadError> { - match resolution { - ResolutionKind::Day => { - get_serialized_line_chart_data::( - db, - chart_name, - from, - to, - points_limit, - policy, - mark_approx, - ) - .await - } - ResolutionKind::Week => { - get_serialized_line_chart_data::( - db, - chart_name, - from, - to, - points_limit, - policy, - mark_approx, - ) - .await - } - ResolutionKind::Month => { - get_serialized_line_chart_data::( - db, - chart_name, - from, - to, - points_limit, - policy, - mark_approx, - ) +impl ReadService { + async fn query_with_handle( + &self, + query_handle: QuerySerializedDyn, + range: UniversalRange>, + points_limit: Option, + query_time: DateTime, + ) -> Result { + let migrations = BlockscoutMigrations::query_from_db(self.blockscout.connection.as_ref()) .await - } - ResolutionKind::Year => { - get_serialized_line_chart_data::( - db, - chart_name, - from, - to, - points_limit, - policy, - mark_approx, - ) + .map_err(ChartError::BlockscoutDB)?; + let context = UpdateContext::from_params_now_or_override(UpdateParameters { + db: &self.db, + blockscout: &self.blockscout, + blockscout_applied_migrations: migrations, + update_time_override: Some(query_time), + force_full: false, + }); + query_handle + .query_data(&context, range, points_limit, true) .await - } + } + + async fn query_counter_with_handle( + &self, + name: String, + settings: EnabledChartSettings, + query_handle: CounterHandle, + query_time: DateTime, + ) -> anyhow::Result { + let point = self + .query_with_handle(query_handle, UniversalRange::full(), None, query_time) + .await?; + Ok(proto_v1::Counter { + id: name.to_string(), + value: point.value, + title: settings.title, + description: settings.description, + units: settings.units, + }) + } + + async fn query_line_chart_with_handle( + &self, + name: String, + chart_entry: &EnabledChartEntry, + query_handle: LineHandle, + range: UniversalRange>, + points_limit: Option, + query_time: DateTime, + ) -> Result { + let data = self + .query_with_handle(query_handle, range, points_limit, query_time) + .await?; + Ok(proto_v1::LineChart { + chart: data, + info: Some(chart_entry.build_proto_line_chart_info(name.to_string())), + }) } } @@ -184,47 +163,45 @@ impl StatsService for ReadService { &self, _request: Request, ) -> Result, Status> { - let mut data = stats::get_raw_counters(&self.db) - .await - .map_err(map_read_error)?; - - let counters = self + let now = Utc::now(); + let counters_futures: FuturesOrdered<_> = self .charts .charts_info .iter() - .filter(|(_, chart)| { - chart - .enabled_resolutions - .iter() - .all(|(_, static_info)| static_info.chart_type == ChartType::Counter) - }) .filter_map(|(name, counter)| { - data.remove(name).and_then(|point| { - // resolutions other than day are currently not supported - // for counters - let Some(static_info) = counter.enabled_resolutions.get(&ResolutionKind::Day) - else { - tracing::warn!( - "No 'day' resolution enabled for counter {}, skipping its value", - name - ); - return None; - }; - let point = if static_info.missing_date_policy == MissingDatePolicy::FillZero { - point.relevant_or_zero(Utc::now().date_naive()) - } else { - point - }; - Some(proto_v1::Counter { - id: static_info.name.clone(), - value: point.value, - title: counter.settings.title.clone(), - description: counter.settings.description.clone(), - units: counter.settings.units.clone(), - }) - }) + // resolutions other than day are currently not supported + // for counters + let Some(enabled_resolution) = counter.resolutions.get(&ResolutionKind::Day) else { + tracing::warn!( + "No 'day' resolution enabled for counter {}, skipping its value", + name + ); + return None; + }; + let query_handle = enabled_resolution + .type_specifics + .clone() + .into_counter_handle()?; + Some(self.query_counter_with_handle( + name.clone(), + counter.settings.clone(), + query_handle, + now, + )) }) .collect(); + let counters: Vec = counters_futures + .filter_map(|query_result| async move { + match query_result { + Ok(c) => Some(c), + Err(e) => { + tracing::error!("Failed to query counter: {:?}", e); + None + } + } + }) + .collect() + .await; let counters_sorted = sorted_items_according_to_layout(counters, &self.charts.counters_layout, |c| &c.id); let counters = proto_v1::Counters { @@ -243,13 +220,16 @@ impl StatsService for ReadService { let chart_entry = self.charts.charts_info.get(&chart_name).ok_or_else(|| { Status::not_found(format!("chart with name '{}' was not found", chart_name)) })?; - let resolution_info = chart_entry - .enabled_resolutions - .get(&resolution) - .filter(|static_info| static_info.chart_type == ChartType::Line) + let resolution_info = chart_entry.resolutions.get(&resolution); + // settings such as `approximate_trailing_points` and `missing_date_policy` + // are used in the type underneath the handle + let query_handle = resolution_info + .and_then(|enabled_resolution| { + enabled_resolution.type_specifics.clone().into_line_handle() + }) .ok_or_else(|| { Status::not_found(format!( - "resolution '{}' for chart '{}' was not found", + "resolution '{}' for line chart '{}' was not found", String::from(resolution), chart_name, )) @@ -257,27 +237,28 @@ impl StatsService for ReadService { let from = request .from - .and_then(|date| NaiveDate::from_str(&date).ok()); - let to = request.to.and_then(|date| NaiveDate::from_str(&date).ok()); - let policy = resolution_info.missing_date_policy; - let mark_approx = resolution_info.approximate_trailing_points; + .and_then(|date| NaiveDate::from_str(&date).ok()) + .map(|d| day_start(&d)); + let to_exclusive = request + .to + .and_then(|date| NaiveDate::from_str(&date).ok()) + .map(|d| day_start(&d.saturating_next_timespan())); + let request_range = (from..to_exclusive).into(); let points_limit = Some(self.limits.requested_points_limit); - let serialized_chart = get_serialized_line_chart_data_resolution_dispatch( - &self.db, - chart_name.clone(), - resolution, - from, - to, - points_limit, - policy, - mark_approx, - ) - .await - .map_err(map_read_error)?; - Ok(Response::new(proto_v1::LineChart { - chart: serialized_chart, - info: Some(chart_entry.build_proto_line_chart_info(chart_name)), - })) + + let chart_data = self + .query_line_chart_with_handle( + chart_name, + chart_entry, + query_handle, + request_range, + points_limit, + Utc::now(), + ) + .await + .map_err(map_update_error)?; + + Ok(Response::new(chart_data)) } async fn get_line_charts( @@ -285,8 +266,7 @@ impl StatsService for ReadService { _request: Request, ) -> Result, Status> { let layout = self.charts.lines_layout.clone(); - let info = self.charts.charts_info.clone(); - let sections = add_chart_info_to_layout(layout, info); + let sections = add_chart_info_to_layout(layout, &self.charts.charts_info); Ok(Response::new(proto_v1::LineCharts { sections })) } diff --git a/stats/stats-server/src/runtime_setup.rs b/stats/stats-server/src/runtime_setup.rs index e257aea6a..d9d07b979 100644 --- a/stats/stats-server/src/runtime_setup.rs +++ b/stats/stats-server/src/runtime_setup.rs @@ -22,8 +22,9 @@ use cron::Schedule; use itertools::Itertools; use stats::{ entity::sea_orm_active_enums::ChartType, + query_dispatch::ChartTypeSpecifics, update_group::{ArcUpdateGroup, SyncUpdateGroup}, - ChartKey, ChartPropertiesObject, ResolutionKind, + ChartKey, ChartObject, ResolutionKind, }; use std::{ collections::{btree_map::Entry, BTreeMap, HashMap, HashSet}, @@ -31,11 +32,11 @@ use std::{ }; use tokio::sync::Mutex; -#[derive(Clone, Debug)] +#[derive(Debug)] pub struct EnabledChartEntry { pub settings: EnabledChartSettings, /// Static information presented as dynamic object - pub enabled_resolutions: HashMap, + pub resolutions: HashMap, } impl EnabledChartEntry { @@ -50,7 +51,7 @@ impl EnabledChartEntry { description: settings.description, units: settings.units, resolutions: self - .enabled_resolutions + .resolutions .keys() .map(|r| String::from(*r)) .collect_vec(), @@ -58,21 +59,25 @@ impl EnabledChartEntry { } } -#[derive(Clone, Debug)] +#[derive(Debug)] pub struct EnabledResolutionEntry { pub name: String, - pub chart_type: ChartType, pub missing_date_policy: stats::MissingDatePolicy, pub approximate_trailing_points: u64, + pub type_specifics: ChartTypeSpecifics, } -impl From for EnabledResolutionEntry { - fn from(value: ChartPropertiesObject) -> Self { +impl From for EnabledResolutionEntry { + fn from(value: ChartObject) -> Self { + let ChartObject { + properties: props, + type_specifics, + } = value; Self { - name: value.name, - chart_type: value.chart_type, - missing_date_policy: value.missing_date_policy, - approximate_trailing_points: value.approximate_trailing_points, + name: props.name, + missing_date_policy: props.missing_date_policy, + approximate_trailing_points: props.approximate_trailing_points, + type_specifics, } } } @@ -111,6 +116,11 @@ fn combine_disjoint_maps( Ok(a) } +struct AllChartsInfo { + counters: BTreeMap, + line_charts: BTreeMap, +} + impl RuntimeSetup { pub fn new( charts: config::charts::Config, @@ -139,10 +149,10 @@ impl RuntimeSetup { /// /// `Err(Vec)` - some unknown charts+resolutions are present in settings fn charts_info_from_settings( + available_resolutions: &mut BTreeMap, charts_settings: BTreeMap, settings_chart_type: ChartType, ) -> Result, Vec> { - let available_resolutions = Self::all_members(); let mut unknown_charts = vec![]; let mut charts_info = BTreeMap::new(); @@ -152,10 +162,14 @@ impl RuntimeSetup { let mut enabled_resolutions_properties = HashMap::new(); for (resolution, resolution_setting) in settings.resolutions.into_list() { let key = ChartKey::new(name.clone(), resolution); - let resolution_properties = available_resolutions - .get(&key) - .filter(|props| props.chart_type == settings_chart_type) - .cloned(); + let resolution_properties = match available_resolutions.entry(key.clone()) { + Entry::Occupied(o) + if o.get().type_specifics.as_chart_type() == settings_chart_type => + { + Some(o.remove()) + } + _ => None, + }; match (resolution_setting, resolution_properties) { // enabled (Some(true), Some(enabled_props)) | (None, Some(enabled_props)) => { @@ -172,7 +186,7 @@ impl RuntimeSetup { name, EnabledChartEntry { settings: enabled_chart_settings, - enabled_resolutions: enabled_resolutions_properties, + resolutions: enabled_resolutions_properties, }, ); } @@ -185,15 +199,26 @@ impl RuntimeSetup { } } - fn build_charts_info( - charts_config: config::charts::Config, - ) -> anyhow::Result> { - let counters_info = - Self::charts_info_from_settings(charts_config.counters, ChartType::Counter); - let lines_info = Self::charts_info_from_settings(charts_config.lines, ChartType::Line); - - let (counters_info, lines_info) = match (counters_info, lines_info) { - (Ok(c), Ok(l)) => (c, l), + fn all_charts_info_from_settings( + counters_settings: BTreeMap, + line_charts_settings: BTreeMap, + ) -> Result> { + let mut available_resolutions = Self::all_members(); + let counters_info = Self::charts_info_from_settings( + &mut available_resolutions, + counters_settings, + ChartType::Counter, + ); + let lines_info = Self::charts_info_from_settings( + &mut available_resolutions, + line_charts_settings, + ChartType::Line, + ); + match (counters_info, lines_info) { + (Ok(c), Ok(l)) => Ok(AllChartsInfo { + counters: c, + line_charts: l, + }), (counters_result, lines_result) => { let mut unknown_charts = vec![]; if let Err(c) = counters_result { @@ -202,13 +227,25 @@ impl RuntimeSetup { if let Err(l) = lines_result { unknown_charts.extend(l); } - return Err(anyhow::anyhow!( - "non-existent charts+resolutions are present in settings: {unknown_charts:?}", - )); + Err(unknown_charts) } - }; + } + } + + fn build_charts_info( + charts_config: config::charts::Config, + ) -> anyhow::Result> { + let AllChartsInfo { + counters, + line_charts, + } = Self::all_charts_info_from_settings(charts_config.counters, charts_config.lines) + .map_err(|unknown_charts| { + anyhow::anyhow!( + "non-existent charts+resolutions are present in settings: {unknown_charts:?}", + ) + })?; - combine_disjoint_maps(counters_info, lines_info) + combine_disjoint_maps(counters, line_charts) .map_err(|duplicate_name| anyhow::anyhow!("duplicate chart name: {duplicate_name:?}",)) } @@ -333,7 +370,7 @@ impl RuntimeSetup { let members: HashSet = group .list_charts() .into_iter() - .map(|c| c.key.as_string()) + .map(|c| c.properties.key.as_string()) .collect(); let missing_members: HashSet = sync_dependencies.difference(&members).cloned().collect(); @@ -393,10 +430,10 @@ impl RuntimeSetup { .into_iter() .filter(|m| { charts_info - .get(m.key.name()) - .is_some_and(|a| a.enabled_resolutions.contains_key(m.key.resolution())) + .get(m.properties.key.name()) + .is_some_and(|a| a.resolutions.contains_key(m.properties.key.resolution())) }) - .map(|m| m.key) + .map(|m| m.properties.key) .collect(); let sync_group = SyncUpdateGroup::new(&dep_mutexes, group)?; result.insert( @@ -412,14 +449,14 @@ impl RuntimeSetup { } /// List all charts+resolutions that are members of at least 1 group. - fn all_members() -> BTreeMap { + fn all_members() -> BTreeMap { let members_with_duplicates = Self::all_update_groups() .into_iter() .flat_map(|g| g.list_charts()) .collect_vec(); let mut members = BTreeMap::new(); for member in members_with_duplicates { - match members.entry(member.key.clone()) { + match members.entry(member.properties.key.clone()) { Entry::Vacant(v) => { v.insert(member); } @@ -430,7 +467,12 @@ impl RuntimeSetup { // i.e. this check does not guarantee that same mutex will not be // used for 2 different charts (although it shouldn't lead to logical // errors) - assert_eq!(o.get(), &member, "duplicate member key '{}'", o.get().key); + assert_eq!( + o.get().properties, + member.properties, + "duplicate member key '{}'", + o.get().properties.key + ); } } } diff --git a/stats/stats-server/src/serializers.rs b/stats/stats-server/src/serializers.rs index f3379bf69..8b1378917 100644 --- a/stats/stats-server/src/serializers.rs +++ b/stats/stats-server/src/serializers.rs @@ -1,23 +1 @@ -use stats::{ - exclusive_datetime_range_to_inclusive, - types::{ExtendedTimespanValue, Timespan}, -}; -use stats_proto::blockscout::stats::v1::Point; -pub fn serialize_line_points( - data: Vec>, -) -> Vec { - data.into_iter() - .map(|point| { - let time_range = - exclusive_datetime_range_to_inclusive(point.timespan.into_time_range()); - let date_range = { time_range.start().date_naive()..=time_range.end().date_naive() }; - Point { - date: date_range.start().to_string(), - date_to: date_range.end().to_string(), - value: point.value, - is_approximate: point.is_approximate, - } - }) - .collect() -} diff --git a/stats/stats-server/src/server.rs b/stats/stats-server/src/server.rs index 0bd45a44f..16f62db3d 100644 --- a/stats/stats-server/src/server.rs +++ b/stats/stats-server/src/server.rs @@ -14,7 +14,7 @@ use anyhow::Context; use blockscout_endpoint_swagger::route_swagger; use blockscout_service_launcher::launcher::{self, LaunchSettings}; use sea_orm::{ConnectOptions, Database}; -use stats::metrics; +use stats::{metrics, utils::MarkedDbConnection}; use stats_proto::blockscout::stats::v1::{ health_actix::route_health, health_server::HealthServer, @@ -79,7 +79,9 @@ pub async fn stats(mut settings: Settings) -> Result<(), anyhow::Error> { settings.run_migrations, ) .await?; - let db = Arc::new(Database::connect(opt).await.context("stats DB")?); + let db = MarkedDbConnection::main_connection(Arc::new( + Database::connect(opt).await.context("stats DB")?, + )); let mut opt = ConnectOptions::new(settings.blockscout_db_url.clone()); opt.sqlx_logging_level(tracing::log::LevelFilter::Debug); @@ -89,7 +91,9 @@ pub async fn stats(mut settings: Settings) -> Result<(), anyhow::Error> { tracing::log::LevelFilter::Warn, Duration::from_secs(3600), ); - let blockscout = Arc::new(Database::connect(opt).await.context("blockscout DB")?); + let blockscout = MarkedDbConnection::main_connection(Arc::new( + Database::connect(opt).await.context("blockscout DB")?, + )); let charts = Arc::new(RuntimeSetup::new( charts_config, @@ -101,14 +105,14 @@ pub async fn stats(mut settings: Settings) -> Result<(), anyhow::Error> { for group_entry in charts.update_groups.values() { group_entry .group - .create_charts_with_mutexes(&db, None, &group_entry.enabled_members) + .create_charts_with_mutexes(db.connection.as_ref(), None, &group_entry.enabled_members) .await?; } let blockscout_api_config = init_blockscout_api_client(&settings).await?; let update_service = - Arc::new(UpdateService::new(db.clone(), blockscout, charts.clone()).await?); + Arc::new(UpdateService::new(db.clone(), blockscout.clone(), charts.clone()).await?); let update_service_handle = tokio::spawn(async move { // Wait for blockscout to index, if necessary. @@ -135,7 +139,8 @@ pub async fn stats(mut settings: Settings) -> Result<(), anyhow::Error> { metrics::initialize_metrics(charts.charts_info.keys().map(|f| f.as_str())); } - let read_service = Arc::new(ReadService::new(db, charts, settings.limits.into()).await?); + let read_service = + Arc::new(ReadService::new(db, blockscout, charts, settings.limits.into()).await?); let health = Arc::new(HealthService::default()); let grpc_router = grpc_router(read_service.clone(), health.clone()); diff --git a/stats/stats-server/src/update_service.rs b/stats/stats-server/src/update_service.rs index 8c05f9a80..543d820f7 100644 --- a/stats/stats-server/src/update_service.rs +++ b/stats/stats-server/src/update_service.rs @@ -1,16 +1,19 @@ use crate::runtime_setup::{RuntimeSetup, UpdateGroupEntry}; use chrono::Utc; use cron::Schedule; -use sea_orm::{DatabaseConnection, DbErr}; -use stats::data_source::types::{BlockscoutMigrations, UpdateParameters}; +use sea_orm::DbErr; +use stats::{ + data_source::types::{BlockscoutMigrations, UpdateParameters}, + utils::MarkedDbConnection, +}; use std::sync::Arc; use tokio::task::JoinHandle; const FAILED_UPDATERS_UNTIL_PANIC: u64 = 3; pub struct UpdateService { - db: Arc, - blockscout: Arc, + db: MarkedDbConnection, + blockscout: MarkedDbConnection, charts: Arc, } @@ -26,8 +29,8 @@ fn time_till_next_call(schedule: &Schedule) -> std::time::Duration { impl UpdateService { pub async fn new( - db: Arc, - blockscout: Arc, + db: MarkedDbConnection, + blockscout: MarkedDbConnection, charts: Arc, ) -> Result { Ok(Self { @@ -104,11 +107,12 @@ impl UpdateService { force_update = force_full, "updating group of charts" ); - let Ok(active_migrations) = BlockscoutMigrations::query_from_db(&self.blockscout) - .await - .inspect_err(|err| { - tracing::error!("error during blockscout migrations detection: {:?}", err) - }) + let Ok(active_migrations) = + BlockscoutMigrations::query_from_db(self.blockscout.connection.as_ref()) + .await + .inspect_err(|err| { + tracing::error!("error during blockscout migrations detection: {:?}", err) + }) else { return; }; diff --git a/stats/stats-server/tests/it/indexing_status.rs b/stats/stats-server/tests/it/indexing_status.rs index d69b072f8..ba6be3887 100644 --- a/stats/stats-server/tests/it/indexing_status.rs +++ b/stats/stats-server/tests/it/indexing_status.rs @@ -85,6 +85,8 @@ async fn test_not_indexed_ok() { } } - let counters: Counters = send_get_request(&base, "/api/v1/counters").await; - assert!(counters.counters.is_empty()) + let mut counters: Counters = send_get_request(&base, "/api/v1/counters").await; + // totalBlocks has fallback with estimate, so it should always return + assert_eq!(counters.counters.pop().unwrap().id, "totalBlocks"); + assert_eq!(counters.counters, vec![]) } diff --git a/stats/stats/Cargo.toml b/stats/stats/Cargo.toml index a99278374..ead654035 100644 --- a/stats/stats/Cargo.toml +++ b/stats/stats/Cargo.toml @@ -14,6 +14,7 @@ sea-orm = { version = "0.12", features = [ tokio = "1" thiserror = "1.0" chrono = "0.4" +trait-variant = { workspace = true } paste = "1.0" portrait = "0.3.0" async-trait = "0.1" @@ -21,6 +22,7 @@ tracing = "0.1" tynm = "0.1.10" futures = "0.3" migration = { path = "./migration" } +stats-proto = { path = "../stats-proto" } url = "2.3" rand = "0.8" lazy_static = "1.4" diff --git a/stats/stats/src/charts/chart.rs b/stats/stats/src/charts/chart.rs index bc1f56d1b..8713a897f 100644 --- a/stats/stats/src/charts/chart.rs +++ b/stats/stats/src/charts/chart.rs @@ -12,10 +12,13 @@ use entity::sea_orm_active_enums::{ChartResolution, ChartType}; use sea_orm::prelude::*; use thiserror::Error; -use super::db_interaction::read::ApproxUnsignedDiff; +use super::{ + db_interaction::read::ApproxUnsignedDiff, + query_dispatch::{ChartTypeSpecifics, QuerySerialized, QuerySerializedDyn}, +}; #[derive(Error, Debug)] -pub enum UpdateError { +pub enum ChartError { #[error("blockscout database error: {0}")] BlockscoutDB(DbErr), #[error("stats database error: {0}")] @@ -28,12 +31,12 @@ pub enum UpdateError { Internal(String), } -impl From for UpdateError { +impl From for ChartError { fn from(read: ReadError) -> Self { match read { - ReadError::DB(db) => UpdateError::StatsDB(db), - ReadError::ChartNotFound(err) => UpdateError::ChartNotFound(err), - ReadError::IntervalTooLarge(limit) => UpdateError::IntervalTooLarge { limit }, + ReadError::DB(db) => ChartError::StatsDB(db), + ReadError::ChartNotFound(err) => ChartError::ChartNotFound(err), + ReadError::IntervalTooLarge(limit) => ChartError::IntervalTooLarge { limit }, } } } @@ -232,6 +235,35 @@ macro_rules! define_and_impl_resolution_properties { }; } +/// Dynamic object representing a chart +#[derive(Debug)] +pub struct ChartObject { + pub properties: ChartPropertiesObject, + pub type_specifics: ChartTypeSpecifics, +} + +impl ChartObject { + pub fn construct_from_chart(t: T) -> Self + where + T: ChartProperties + QuerySerialized + Send + 'static, + QuerySerializedDyn: Into, + { + let type_specifics = as Into>::into( + std::sync::Arc::new(Box::new(t)), + ); + assert_eq!( + type_specifics.as_chart_type(), + T::chart_type(), + "data returned by chart {} does not match chart type", + T::name() + ); + Self { + properties: ChartPropertiesObject::construct_from_chart::(), + type_specifics, + } + } +} + /// Dynamic version of trait `ChartProperties`. /// /// Helpful when need a unified type for different charts @@ -240,7 +272,6 @@ pub struct ChartPropertiesObject { /// unique identifier of the chart pub key: ChartKey, pub name: String, - pub chart_type: ChartType, pub resolution: ResolutionKind, pub missing_date_policy: MissingDatePolicy, pub approximate_trailing_points: u64, @@ -251,7 +282,6 @@ impl ChartPropertiesObject { Self { key: T::key(), name: T::name(), - chart_type: T::chart_type(), resolution: T::resolution(), missing_date_policy: T::missing_date_policy(), approximate_trailing_points: T::approximate_trailing_points(), diff --git a/stats/stats/src/charts/counters/average_block_time.rs b/stats/stats/src/charts/counters/average_block_time.rs index 0e4fb595f..3d865cc36 100644 --- a/stats/stats/src/charts/counters/average_block_time.rs +++ b/stats/stats/src/charts/counters/average_block_time.rs @@ -1,4 +1,4 @@ -use std::{cmp::Reverse, ops::Range}; +use std::cmp::Reverse; use crate::{ data_source::{ @@ -9,13 +9,14 @@ use crate::{ }, UpdateContext, }, + range::UniversalRange, types::TimespanValue, utils::NANOS_PER_SEC, - ChartProperties, MissingDatePolicy, Named, UpdateError, + ChartError, ChartProperties, MissingDatePolicy, Named, }; use blockscout_db::entity::blocks; -use chrono::NaiveDate; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; use itertools::Itertools; use sea_orm::{prelude::*, DbBackend, FromQueryResult, QueryOrder, QuerySelect, Statement}; @@ -51,12 +52,12 @@ struct BlockTimestamp { async fn query_average_block_time( cx: &UpdateContext<'_>, offset: u64, -) -> Result>, UpdateError> { +) -> Result>, ChartError> { let query = average_block_time_statement(offset); let block_timestamps = BlockTimestamp::find_by_statement(query) - .all(cx.blockscout) + .all(cx.blockscout.connection.as_ref()) .await - .map_err(UpdateError::BlockscoutDB)?; + .map_err(ChartError::BlockscoutDB)?; Ok(calculate_average_block_time(block_timestamps)) } @@ -67,13 +68,13 @@ impl RemoteQueryBehaviour for AverageBlockTimeQuery { async fn query_data( cx: &UpdateContext<'_>, - _range: Option>, - ) -> Result, UpdateError> { + _range: UniversalRange>, + ) -> Result, ChartError> { match query_average_block_time(cx, OFFSET_BLOCKS).await? { Some(avg_block_time) => Ok(avg_block_time), None => query_average_block_time(cx, 0) .await? - .ok_or(UpdateError::Internal( + .ok_or(ChartError::Internal( "No blocks were returned to calculate average block time".into(), )), } @@ -148,6 +149,7 @@ mod tests { mock_blockscout::fill_many_blocks, simple_test::{get_counter, prepare_chart_test, simple_test_counter}, }, + utils::MarkedDbConnection, }; #[tokio::test] @@ -196,8 +198,8 @@ mod tests { }; fill_many_blocks(&blockscout, current_time.naive_utc(), &block_times).await; let mut parameters = UpdateParameters { - db: &db, - blockscout: &blockscout, + db: &MarkedDbConnection::from_test_db(&db).unwrap(), + blockscout: &MarkedDbConnection::from_test_db(&blockscout).unwrap(), blockscout_applied_migrations: BlockscoutMigrations::latest(), update_time_override: Some(current_time), force_full: true, @@ -206,14 +208,14 @@ mod tests { AverageBlockTime::update_recursively(&cx).await.unwrap(); assert_eq!( expected_avg.to_string(), - get_counter::(&db).await + get_counter::(&cx).await.value ); parameters.force_full = false; let cx = UpdateContext::from_params_now_or_override(parameters.clone()); AverageBlockTime::update_recursively(&cx).await.unwrap(); assert_eq!( expected_avg.to_string(), - get_counter::(&db).await + get_counter::(&cx).await.value ); } diff --git a/stats/stats/src/charts/counters/last_new_contracts.rs b/stats/stats/src/charts/counters/last_new_contracts.rs index 6298eb78a..efad07605 100644 --- a/stats/stats/src/charts/counters/last_new_contracts.rs +++ b/stats/stats/src/charts/counters/last_new_contracts.rs @@ -1,6 +1,7 @@ use crate::{ data_source::kinds::{ - data_manipulation::last_point::LastPoint, local_db::DirectPointLocalDbChartSource, + data_manipulation::{last_point::LastPoint, map::StripExt}, + local_db::DirectPointLocalDbChartSource, }, lines::NewContracts, ChartProperties, Named, @@ -25,7 +26,8 @@ impl ChartProperties for Properties { } } -pub type LastNewContracts = DirectPointLocalDbChartSource, Properties>; +pub type LastNewContracts = + DirectPointLocalDbChartSource>, Properties>; #[cfg(test)] mod tests { diff --git a/stats/stats/src/charts/counters/last_new_verified_contracts.rs b/stats/stats/src/charts/counters/last_new_verified_contracts.rs index bae0a5650..271af6f0f 100644 --- a/stats/stats/src/charts/counters/last_new_verified_contracts.rs +++ b/stats/stats/src/charts/counters/last_new_verified_contracts.rs @@ -1,6 +1,7 @@ use crate::{ data_source::kinds::{ - data_manipulation::last_point::LastPoint, local_db::DirectPointLocalDbChartSource, + data_manipulation::{last_point::LastPoint, map::StripExt}, + local_db::DirectPointLocalDbChartSource, }, lines::NewVerifiedContracts, ChartProperties, Named, @@ -26,7 +27,7 @@ impl ChartProperties for Properties { } pub type LastNewVerifiedContracts = - DirectPointLocalDbChartSource, Properties>; + DirectPointLocalDbChartSource>, Properties>; #[cfg(test)] mod tests { diff --git a/stats/stats/src/charts/counters/mock.rs b/stats/stats/src/charts/counters/mock.rs index 2c138da64..d29b51d1c 100644 --- a/stats/stats/src/charts/counters/mock.rs +++ b/stats/stats/src/charts/counters/mock.rs @@ -1,4 +1,4 @@ -use std::{marker::PhantomData, ops::Range}; +use std::marker::PhantomData; use crate::{ data_source::{ @@ -9,13 +9,13 @@ use crate::{ types::Get, UpdateContext, }, + range::UniversalRange, types::timespans::DateValue, - ChartProperties, Named, UpdateError, + ChartError, ChartProperties, Named, }; use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::prelude::DateTimeUtc; pub struct MockCounterRetrieve(PhantomData<(PointDateTime, Value)>) where @@ -31,8 +31,8 @@ where async fn query_data( cx: &UpdateContext<'_>, - _range: Option>, - ) -> Result { + _range: UniversalRange>, + ) -> Result { if cx.time >= PointDateTime::get() { Ok(DateValue:: { timespan: PointDateTime::get().date_naive(), diff --git a/stats/stats/src/charts/counters/total_accounts.rs b/stats/stats/src/charts/counters/total_accounts.rs index 97c9fd8d3..2ff88c84f 100644 --- a/stats/stats/src/charts/counters/total_accounts.rs +++ b/stats/stats/src/charts/counters/total_accounts.rs @@ -1,6 +1,7 @@ use crate::{ data_source::kinds::{ - data_manipulation::last_point::LastPoint, local_db::DirectPointLocalDbChartSource, + data_manipulation::{last_point::LastPoint, map::StripExt}, + local_db::DirectPointLocalDbChartSource, }, lines::AccountsGrowth, ChartProperties, MissingDatePolicy, Named, @@ -28,7 +29,8 @@ impl ChartProperties for Properties { } } -pub type TotalAccounts = DirectPointLocalDbChartSource, Properties>; +pub type TotalAccounts = + DirectPointLocalDbChartSource>, Properties>; #[cfg(test)] mod tests { diff --git a/stats/stats/src/charts/counters/total_blocks.rs b/stats/stats/src/charts/counters/total_blocks.rs index 8b14fb609..4bca4811b 100644 --- a/stats/stats/src/charts/counters/total_blocks.rs +++ b/stats/stats/src/charts/counters/total_blocks.rs @@ -1,19 +1,20 @@ -use std::ops::Range; - use crate::{ + charts::db_interaction::read::query_estimated_table_rows, data_source::{ kinds::{ - local_db::DirectPointLocalDbChartSource, + local_db::{parameters::ValueEstimation, DirectPointLocalDbChartSourceWithEstimate}, remote_db::{RemoteDatabaseSource, RemoteQueryBehaviour}, }, types::UpdateContext, }, + range::UniversalRange, types::timespans::DateValue, - ChartProperties, MissingDatePolicy, Named, UpdateError, + utils::MarkedDbConnection, + ChartError, ChartProperties, MissingDatePolicy, Named, }; use blockscout_db::entity::blocks; -use chrono::{NaiveDate, NaiveDateTime}; +use chrono::{DateTime, NaiveDate, NaiveDateTime, Utc}; use entity::sea_orm_active_enums::ChartType; use sea_orm::{prelude::*, sea_query::Expr, FromQueryResult, QuerySelect}; @@ -30,18 +31,18 @@ impl RemoteQueryBehaviour for TotalBlocksQueryBehaviour { async fn query_data( cx: &UpdateContext<'_>, - _range: Option>, - ) -> Result { + _range: UniversalRange>, + ) -> Result { let data = blocks::Entity::find() .select_only() .column_as(Expr::col(blocks::Column::Number).count(), "number") .column_as(Expr::col(blocks::Column::Timestamp).max(), "timestamp") .filter(blocks::Column::Consensus.eq(true)) .into_model::() - .one(cx.blockscout) + .one(cx.blockscout.connection.as_ref()) .await - .map_err(UpdateError::BlockscoutDB)? - .ok_or_else(|| UpdateError::Internal("query returned nothing".into()))?; + .map_err(ChartError::BlockscoutDB)? + .ok_or_else(|| ChartError::Internal("query returned nothing".into()))?; let data = DateValue:: { timespan: data.timestamp.date(), @@ -72,32 +73,55 @@ impl ChartProperties for Properties { } } -pub type TotalBlocks = DirectPointLocalDbChartSource; +pub struct TotalBlocksEstimation; + +impl ValueEstimation for TotalBlocksEstimation { + async fn estimate(blockscout: &MarkedDbConnection) -> Result, ChartError> { + let now = Utc::now(); + let value = + query_estimated_table_rows(blockscout.connection.as_ref(), blocks::Entity.table_name()) + .await + .map_err(ChartError::BlockscoutDB)? + .map(|b| { + let b = b as f64 * 0.9; + b as i64 + }) + .unwrap_or(0); + Ok(DateValue { + timespan: now.date_naive(), + value: value.to_string(), + }) + } +} + +pub type TotalBlocks = + DirectPointLocalDbChartSourceWithEstimate; #[cfg(test)] mod tests { use super::*; use crate::{ data_source::{types::BlockscoutMigrations, DataSource, UpdateContext, UpdateParameters}, - get_raw_counters, - tests::{init_db::init_db_all, mock_blockscout::fill_mock_blockscout_data}, - Named, + tests::{ + init_db::init_marked_db_all, mock_blockscout::fill_mock_blockscout_data, + simple_test::get_counter, + }, }; use chrono::NaiveDate; use entity::chart_data; - use pretty_assertions::assert_eq; - use sea_orm::{DatabaseConnection, EntityTrait, Set}; + use pretty_assertions::{assert_eq, assert_ne}; + use sea_orm::{DatabaseConnection, DbBackend, EntityTrait, Set, Statement}; use std::str::FromStr; #[tokio::test] #[ignore = "needs database to run"] async fn update_total_blocks_recurrent() { let _ = tracing_subscriber::fmt::try_init(); - let (db, blockscout) = init_db_all("update_total_blocks_recurrent").await; + let (db, blockscout) = init_marked_db_all("update_total_blocks_recurrent").await; let current_time = chrono::DateTime::from_str("2023-03-01T12:00:00Z").unwrap(); let current_date = current_time.date_naive(); - TotalBlocks::init_recursively(&db, ¤t_time) + TotalBlocks::init_recursively(&db.connection, ¤t_time) .await .unwrap(); @@ -107,11 +131,11 @@ mod tests { value: Set(1.to_string()), ..Default::default() }) - .exec(&db as &DatabaseConnection) + .exec(&db.connection as &DatabaseConnection) .await .unwrap(); - fill_mock_blockscout_data(&blockscout, current_date).await; + fill_mock_blockscout_data(&blockscout.connection, current_date).await; let parameters = UpdateParameters { db: &db, @@ -122,23 +146,23 @@ mod tests { }; let cx = UpdateContext::from_params_now_or_override(parameters.clone()); TotalBlocks::update_recursively(&cx).await.unwrap(); - let data = get_raw_counters(&db).await.unwrap(); - assert_eq!("13", data[&TotalBlocks::name()].value); + let data = get_counter::(&cx).await; + assert_eq!("13", data.value); } #[tokio::test] #[ignore = "needs database to run"] async fn update_total_blocks_fresh() { let _ = tracing_subscriber::fmt::try_init(); - let (db, blockscout) = init_db_all("update_total_blocks_fresh").await; + let (db, blockscout) = init_marked_db_all("update_total_blocks_fresh").await; let current_time = chrono::DateTime::from_str("2022-11-12T12:00:00Z").unwrap(); let current_date = current_time.date_naive(); - TotalBlocks::init_recursively(&db, ¤t_time) + TotalBlocks::init_recursively(&db.connection, ¤t_time) .await .unwrap(); - fill_mock_blockscout_data(&blockscout, current_date).await; + fill_mock_blockscout_data(&blockscout.connection, current_date).await; let parameters = UpdateParameters { db: &db, @@ -149,19 +173,19 @@ mod tests { }; let cx = UpdateContext::from_params_now_or_override(parameters.clone()); TotalBlocks::update_recursively(&cx).await.unwrap(); - let data = get_raw_counters(&db).await.unwrap(); - assert_eq!("9", data[&TotalBlocks::name()].value); + let data = get_counter::(&cx).await; + assert_eq!("9", data.value); } #[tokio::test] #[ignore = "needs database to run"] async fn update_total_blocks_last() { let _ = tracing_subscriber::fmt::try_init(); - let (db, blockscout) = init_db_all("update_total_blocks_last").await; + let (db, blockscout) = init_marked_db_all("update_total_blocks_last").await; let current_time = chrono::DateTime::from_str("2023-03-01T12:00:00Z").unwrap(); let current_date = current_time.date_naive(); - TotalBlocks::init_recursively(&db, ¤t_time) + TotalBlocks::init_recursively(&db.connection, ¤t_time) .await .unwrap(); @@ -171,11 +195,11 @@ mod tests { value: Set(1.to_string()), ..Default::default() }) - .exec(&db as &DatabaseConnection) + .exec(&db.connection as &DatabaseConnection) .await .unwrap(); - fill_mock_blockscout_data(&blockscout, current_date).await; + fill_mock_blockscout_data(&blockscout.connection, current_date).await; let parameters = UpdateParameters { db: &db, @@ -186,7 +210,41 @@ mod tests { }; let cx = UpdateContext::from_params_now_or_override(parameters.clone()); TotalBlocks::update_recursively(&cx).await.unwrap(); - let data = get_raw_counters(&db).await.unwrap(); - assert_eq!("13", data[&TotalBlocks::name()].value); + let data = get_counter::(&cx).await; + assert_eq!("13", data.value); + } + + #[tokio::test] + #[ignore = "needs database to run"] + async fn total_blocks_fallback() { + let _ = tracing_subscriber::fmt::try_init(); + let (db, blockscout) = init_marked_db_all("total_blocks_fallback").await; + let current_time = chrono::DateTime::from_str("2023-03-01T12:00:00Z").unwrap(); + let current_date = current_time.date_naive(); + + TotalBlocks::init_recursively(&db.connection, ¤t_time) + .await + .unwrap(); + + fill_mock_blockscout_data(&blockscout.connection, current_date).await; + + // need to analyze or vacuum for `reltuples` to be updated. + // source: https://www.postgresql.org/docs/9.3/planner-stats.html + let _ = blockscout + .connection + .execute(Statement::from_string(DbBackend::Postgres, "ANALYZE;")) + .await + .unwrap(); + + let parameters = UpdateParameters { + db: &db, + blockscout: &blockscout, + blockscout_applied_migrations: BlockscoutMigrations::latest(), + update_time_override: Some(current_time), + force_full: false, + }; + let cx: UpdateContext<'_> = UpdateContext::from_params_now_or_override(parameters.clone()); + let data = get_counter::(&cx).await; + assert_ne!("0", data.value); } } diff --git a/stats/stats/src/charts/counters/total_contracts.rs b/stats/stats/src/charts/counters/total_contracts.rs index 1bba5fb43..841d7b6a1 100644 --- a/stats/stats/src/charts/counters/total_contracts.rs +++ b/stats/stats/src/charts/counters/total_contracts.rs @@ -1,5 +1,3 @@ -use std::ops::Range; - use crate::{ data_source::{ kinds::{ @@ -8,12 +6,13 @@ use crate::{ }, UpdateContext, }, + range::UniversalRange, types::timespans::DateValue, - ChartProperties, MissingDatePolicy, Named, UpdateError, + ChartError, ChartProperties, MissingDatePolicy, Named, }; use blockscout_db::entity::addresses; -use chrono::NaiveDate; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; use sea_orm::prelude::*; @@ -24,14 +23,14 @@ impl RemoteQueryBehaviour for TotalContractsQueryBehaviour { async fn query_data( cx: &UpdateContext<'_>, - _range: Option>, - ) -> Result { + _range: UniversalRange>, + ) -> Result { let value = addresses::Entity::find() .filter(addresses::Column::ContractCode.is_not_null()) .filter(addresses::Column::InsertedAt.lte(cx.time)) - .count(cx.blockscout) + .count(cx.blockscout.connection.as_ref()) .await - .map_err(UpdateError::BlockscoutDB)?; + .map_err(ChartError::BlockscoutDB)?; let timespan = cx.time.date_naive(); Ok(DateValue:: { timespan, diff --git a/stats/stats/src/charts/counters/total_native_coin_holders.rs b/stats/stats/src/charts/counters/total_native_coin_holders.rs index 3e40e42fc..d87acb5d2 100644 --- a/stats/stats/src/charts/counters/total_native_coin_holders.rs +++ b/stats/stats/src/charts/counters/total_native_coin_holders.rs @@ -1,6 +1,7 @@ use crate::{ data_source::kinds::{ - data_manipulation::last_point::LastPoint, local_db::DirectPointLocalDbChartSource, + data_manipulation::{last_point::LastPoint, map::StripExt}, + local_db::DirectPointLocalDbChartSource, }, lines::NativeCoinHoldersGrowth, ChartProperties, MissingDatePolicy, Named, @@ -29,7 +30,7 @@ impl ChartProperties for Properties { } pub type TotalNativeCoinHolders = - DirectPointLocalDbChartSource, Properties>; + DirectPointLocalDbChartSource>, Properties>; #[cfg(test)] mod tests { diff --git a/stats/stats/src/charts/counters/total_verified_contracts.rs b/stats/stats/src/charts/counters/total_verified_contracts.rs index 38eb0880d..6acf9712c 100644 --- a/stats/stats/src/charts/counters/total_verified_contracts.rs +++ b/stats/stats/src/charts/counters/total_verified_contracts.rs @@ -1,6 +1,7 @@ use crate::{ data_source::kinds::{ - data_manipulation::last_point::LastPoint, local_db::DirectPointLocalDbChartSource, + data_manipulation::{last_point::LastPoint, map::StripExt}, + local_db::DirectPointLocalDbChartSource, }, lines::VerifiedContractsGrowth, ChartProperties, MissingDatePolicy, Named, @@ -29,7 +30,7 @@ impl ChartProperties for Properties { } pub type TotalVerifiedContracts = - DirectPointLocalDbChartSource, Properties>; + DirectPointLocalDbChartSource>, Properties>; #[cfg(test)] mod tests { diff --git a/stats/stats/src/charts/counters/yesterday_txns.rs b/stats/stats/src/charts/counters/yesterday_txns.rs index 6ef5131dc..691ee7b97 100644 --- a/stats/stats/src/charts/counters/yesterday_txns.rs +++ b/stats/stats/src/charts/counters/yesterday_txns.rs @@ -1,5 +1,3 @@ -use std::ops::Range; - use crate::{ data_source::{ kinds::{ @@ -9,9 +7,10 @@ use crate::{ UpdateContext, }, lines::NewTxnsStatement, + range::UniversalRange, types::TimespanValue, utils::day_start, - ChartProperties, MissingDatePolicy, Named, UpdateError, + ChartError, ChartProperties, MissingDatePolicy, Named, }; use chrono::{DateTime, Days, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; @@ -24,12 +23,12 @@ impl RemoteQueryBehaviour for YesterdayTxnsQuery { async fn query_data( cx: &UpdateContext<'_>, - _range: Option>>, - ) -> Result { + _range: UniversalRange>, + ) -> Result { let today = cx.time.date_naive(); let yesterday = today .checked_sub_days(Days::new(1)) - .ok_or(UpdateError::Internal( + .ok_or(ChartError::Internal( "Update time is incorrect: ~ minimum possible date".into(), ))?; let yesterday_range = day_start(&yesterday)..day_start(&today); @@ -38,9 +37,9 @@ impl RemoteQueryBehaviour for YesterdayTxnsQuery { &cx.blockscout_applied_migrations, ); let data = Self::Output::find_by_statement(query) - .one(cx.blockscout) + .one(cx.blockscout.connection.as_ref()) .await - .map_err(UpdateError::BlockscoutDB)? + .map_err(ChartError::BlockscoutDB)? // no transactions for yesterday .unwrap_or(TimespanValue::with_zero_value(yesterday)); Ok(data) diff --git a/stats/stats/src/charts/db_interaction/read.rs b/stats/stats/src/charts/db_interaction/read.rs index ef490c2cb..ac67a2191 100644 --- a/stats/stats/src/charts/db_interaction/read.rs +++ b/stats/stats/src/charts/db_interaction/read.rs @@ -5,12 +5,12 @@ use crate::{ UpdateContext, }, missing_date::{fill_and_filter_chart, fit_into_range}, + range::{exclusive_range_to_inclusive, UniversalRange}, types::{ timespans::{DateValue, Month, Week, Year}, ExtendedTimespanValue, Timespan, TimespanDuration, TimespanValue, }, - utils::exclusive_datetime_range_to_inclusive, - ChartProperties, MissingDatePolicy, UpdateError, + ChartError, ChartProperties, MissingDatePolicy, }; use blockscout_db::entity::blocks; @@ -22,7 +22,7 @@ use sea_orm::{ ColumnTrait, ConnectionTrait, DatabaseConnection, DbBackend, DbErr, EntityTrait, FromQueryResult, QueryFilter, QueryOrder, QuerySelect, Statement, }; -use std::{collections::HashMap, fmt::Debug, ops::Range}; +use std::{fmt::Debug, ops::Range}; use thiserror::Error; use tracing::instrument; @@ -55,48 +55,6 @@ pub async fn find_chart(db: &DatabaseConnection, chart: &ChartKey) -> Result Result>, ReadError> { - let data = CounterData::find_by_statement(Statement::from_string( - DbBackend::Postgres, - r#" - SELECT distinct on (charts.id) charts.name, data.date, data.value - FROM "chart_data" "data" - INNER JOIN "charts" - ON data.chart_id = charts.id - WHERE charts.chart_type = 'COUNTER' - ORDER BY charts.id, data.id DESC; - "#, - )) - .all(db) - .await?; - - let counters: HashMap<_, _> = data - .into_iter() - .map(|data| { - ( - data.name, - DateValue:: { - timespan: data.date, - value: data.value, - }, - ) - }) - .collect(); - - Ok(counters) -} - /// Get counter value for the requested date pub async fn get_counter_data( db: &DatabaseConnection, @@ -196,7 +154,7 @@ fn relevant_data_until( R::from_date(t.date_naive()).saturating_start_timestamp() == t; // last_updated_at timestamp is not included in the range let inclusive_last_updated_at_end = - exclusive_datetime_range_to_inclusive(DateTime::::MIN_UTC..t); + exclusive_range_to_inclusive(DateTime::::MIN_UTC..t); ( Some(R::from_date( inclusive_last_updated_at_end.end().date_naive(), @@ -469,6 +427,45 @@ where .ok_or_else(|| DbErr::RecordNotFound("no blocks found in blockscout database".into())) } +#[derive(Debug, FromQueryResult)] +struct CountEstimate { + count: Option, +} + +/// `None` means either that +/// - db hasn't been initialized before +/// - `table_name` wasn't found +pub async fn query_estimated_table_rows( + blockscout: &DatabaseConnection, + table_name: &str, +) -> Result, DbErr> { + let statement: Statement = Statement::from_sql_and_values( + DbBackend::Postgres, + r#" + SELECT ( + CASE WHEN c.reltuples < 0 THEN + NULL + WHEN c.relpages = 0 THEN + float8 '0' + ELSE c.reltuples / c.relpages + END * + ( + pg_catalog.pg_relation_size(c.oid) / + pg_catalog.current_setting('block_size')::int + ) + )::bigint as count + FROM pg_catalog.pg_class c + WHERE c.oid = $1::regclass + "#, + vec![table_name.into()], + ); + let count = CountEstimate::find_by_statement(statement) + .one(blockscout) + .await?; + let count = count.and_then(|c| c.count); + Ok(count) +} + #[derive(Debug, FromQueryResult)] struct SyncInfo { pub min_blockscout_block: Option, @@ -485,7 +482,7 @@ pub async fn last_accurate_point( force_full: bool, approximate_trailing_points: u64, policy: MissingDatePolicy, -) -> Result>, UpdateError> +) -> Result>, ChartError> where ChartProps: ChartProperties + ?Sized, ChartProps::Resolution: Ord + Clone + Debug, @@ -502,7 +499,7 @@ where .into_model() .one(db) .await - .map_err(UpdateError::StatsDB)?; + .map_err(ChartError::StatsDB)?; let metadata = get_chart_metadata(db, &ChartProps::key()).await?; match recorded_min_blockscout_block { @@ -539,7 +536,7 @@ where } }); let Some(last_accurate_point) = last_accurate_point else { - return Err(UpdateError::Internal("Failure while reading chart data: did not return accurate data (with `fill_missing_dates`=true)".into())); + return Err(ChartError::Internal("Failure while reading chart data: did not return accurate data (with `fill_missing_dates`=true)".into())); }; if let Some(block) = recorded_min_blockscout_block.min_blockscout_block { @@ -658,11 +655,11 @@ impl RemoteQueryBehaviour for QueryAllBlockTimestampRange { async fn query_data( cx: &UpdateContext<'_>, - _range: Option>>, - ) -> Result { - let start_timestamp = get_min_date_blockscout(cx.blockscout) + _range: UniversalRange>, + ) -> Result { + let start_timestamp = get_min_date_blockscout(cx.blockscout.connection.as_ref()) .await - .map_err(UpdateError::BlockscoutDB)? + .map_err(ChartError::BlockscoutDB)? .and_utc(); Ok(start_timestamp..cx.time) } @@ -674,19 +671,25 @@ mod tests { use crate::{ charts::ResolutionKind, counters::TotalBlocks, - data_source::kinds::local_db::parameters::DefaultQueryVec, + data_source::{ + kinds::local_db::parameters::DefaultQueryVec, types::BlockscoutMigrations, + UpdateParameters, + }, lines::{AccountsGrowth, ActiveAccounts, TxnsGrowth, TxnsGrowthMonthly}, tests::{ - init_db::init_db, - point_construction::{d, month_of}, + init_db::{init_db, init_db_all}, + mock_blockscout::fill_mock_blockscout_data, + point_construction::{d, dt, month_of}, + simple_test::get_counter, }, types::timespans::Month, + utils::MarkedDbConnection, Named, }; use chrono::DateTime; use entity::{chart_data, charts, sea_orm_active_enums::ChartType}; use pretty_assertions::assert_eq; - use sea_orm::{EntityTrait, Set}; + use sea_orm::{EntityName, EntityTrait, Set}; use std::str::FromStr; fn mock_chart_data(chart_id: i32, date: &str, value: i64) -> chart_data::ActiveModel { @@ -823,15 +826,24 @@ mod tests { #[tokio::test] #[ignore = "needs database to run"] - async fn get_counters_mock() { + async fn get_counter_mock() { let _ = tracing_subscriber::fmt::try_init(); - let db = init_db("get_counters_mock").await; - insert_mock_data(&db).await; - let counters = get_raw_counters(&db).await.unwrap(); + let db = MarkedDbConnection::from_test_db(&init_db("get_counter_mock").await).unwrap(); + insert_mock_data(&db.connection).await; + let current_time = dt("2022-11-12T08:08:08").and_utc(); + let date = current_time.date_naive(); + let cx = UpdateContext::from_params_now_or_override(UpdateParameters { + db: &db, + // shouldn't use this because mock data contains total blocks value + blockscout: &db, + blockscout_applied_migrations: BlockscoutMigrations::latest(), + update_time_override: Some(current_time), + force_full: false, + }); assert_eq!( - HashMap::from_iter([("totalBlocks".into(), value("2022-11-12", "1350"))]), - counters + value(&date.to_string(), "1350"), + get_counter::(&cx).await ); } @@ -1290,4 +1302,60 @@ mod tests { }) ); } + + #[tokio::test] + #[ignore = "needs database to run"] + async fn get_estimated_table_rows_works() { + let (_db, blockscout) = init_db_all("get_estimated_table_rows_works").await; + fill_mock_blockscout_data(&blockscout, d("2023-03-01")).await; + + // need to analyze or vacuum for `reltuples` to be updated. + // source: https://www.postgresql.org/docs/9.3/planner-stats.html + let _ = blockscout + .execute(Statement::from_string(DbBackend::Postgres, "ANALYZE;")) + .await + .unwrap(); + + let blocks_estimate = query_estimated_table_rows(&blockscout, blocks::Entity.table_name()) + .await + .unwrap() + .unwrap(); + + // should be 16 rows in the table, but it's an estimate + assert!(blocks_estimate > 5); + assert!(blocks_estimate < 30); + + assert!( + query_estimated_table_rows( + &blockscout, + blockscout_db::entity::addresses::Entity.table_name() + ) + .await + .unwrap() + .unwrap() + > 0 + ); + + assert!( + query_estimated_table_rows( + &blockscout, + blockscout_db::entity::transactions::Entity.table_name() + ) + .await + .unwrap() + .unwrap() + > 0 + ); + + assert!( + query_estimated_table_rows( + &blockscout, + blockscout_db::entity::smart_contracts::Entity.table_name() + ) + .await + .unwrap() + .unwrap() + > 0 + ); + } } diff --git a/stats/stats/src/charts/db_interaction/write.rs b/stats/stats/src/charts/db_interaction/write.rs index d70dd38a5..f5706889d 100644 --- a/stats/stats/src/charts/db_interaction/write.rs +++ b/stats/stats/src/charts/db_interaction/write.rs @@ -1,6 +1,6 @@ use chrono::{DateTime, Offset, TimeZone}; use entity::{chart_data, charts, sea_orm_active_enums::ChartType}; -use sea_orm::{prelude::*, sea_query, ConnectionTrait, Set, Unchanged}; +use sea_orm::{prelude::*, sea_query, Set, Unchanged}; use crate::charts::ChartKey; diff --git a/stats/stats/src/charts/lines/accounts_growth.rs b/stats/stats/src/charts/lines/accounts_growth.rs index 6c5410402..246c3b5ce 100644 --- a/stats/stats/src/charts/lines/accounts_growth.rs +++ b/stats/stats/src/charts/lines/accounts_growth.rs @@ -3,7 +3,7 @@ use super::new_accounts::NewAccountsInt; use crate::{ data_source::kinds::{ - data_manipulation::resolutions::last_value::LastValueLowerResolution, + data_manipulation::{map::StripExt, resolutions::last_value::LastValueLowerResolution}, local_db::{ parameters::update::batching::parameters::{Batch30Weeks, Batch30Years, Batch36Months}, DailyCumulativeLocalDbChartSource, DirectVecLocalDbChartSource, @@ -46,18 +46,21 @@ define_and_impl_resolution_properties!( ); pub type AccountsGrowth = DailyCumulativeLocalDbChartSource; +type AccountsGrowthS = StripExt; + pub type AccountsGrowthWeekly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch30Weeks, WeeklyProperties, >; pub type AccountsGrowthMonthly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch36Months, MonthlyProperties, >; +type AccountsGrowthMonthlyS = StripExt; pub type AccountsGrowthYearly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch30Years, YearlyProperties, >; diff --git a/stats/stats/src/charts/lines/active_accounts.rs b/stats/stats/src/charts/lines/active_accounts.rs index 51162792f..86172eba8 100644 --- a/stats/stats/src/charts/lines/active_accounts.rs +++ b/stats/stats/src/charts/lines/active_accounts.rs @@ -3,6 +3,7 @@ use std::ops::Range; use crate::{ + charts::db_interaction::read::QueryAllBlockTimestampRange, data_source::{ kinds::{ local_db::{ @@ -16,15 +17,15 @@ use crate::{ ChartProperties, Named, }; -use chrono::NaiveDate; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{prelude::*, DbBackend, Statement}; +use sea_orm::{DbBackend, Statement}; pub struct ActiveAccountsStatement; impl StatementFromRange for ActiveAccountsStatement { fn get_statement( - range: Option>, + range: Option>>, completed_migrations: &BlockscoutMigrations, ) -> Statement { if completed_migrations.denormalization { @@ -66,8 +67,9 @@ impl StatementFromRange for ActiveAccountsStatement { } } -pub type ActiveAccountsRemote = - RemoteDatabaseSource>; +pub type ActiveAccountsRemote = RemoteDatabaseSource< + PullAllWithAndSort, +>; pub struct Properties; diff --git a/stats/stats/src/charts/lines/average_block_rewards.rs b/stats/stats/src/charts/lines/average_block_rewards.rs index 7377eb225..d14357f73 100644 --- a/stats/stats/src/charts/lines/average_block_rewards.rs +++ b/stats/stats/src/charts/lines/average_block_rewards.rs @@ -1,10 +1,11 @@ use std::ops::Range; use crate::{ + charts::db_interaction::read::QueryAllBlockTimestampRange, data_source::{ kinds::{ data_manipulation::{ - map::{MapParseTo, MapToString}, + map::{MapParseTo, MapToString, StripExt}, resolutions::average::AverageLowerResolution, }, local_db::{ @@ -23,9 +24,9 @@ use crate::{ ChartProperties, Named, }; -use chrono::NaiveDate; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{prelude::*, DbBackend, Statement}; +use sea_orm::{DbBackend, Statement}; use super::{NewBlockRewardsInt, NewBlockRewardsMonthlyInt}; @@ -34,7 +35,7 @@ const ETH: i64 = 1_000_000_000_000_000_000; pub struct AverageBlockRewardsQuery; impl StatementFromRange for AverageBlockRewardsQuery { - fn get_statement(range: Option>, _: &BlockscoutMigrations) -> Statement { + fn get_statement(range: Option>>, _: &BlockscoutMigrations) -> Statement { sql_with_range_filter_opt!( DbBackend::Postgres, r#" @@ -55,8 +56,9 @@ impl StatementFromRange for AverageBlockRewardsQuery { } } -pub type AverageBlockRewardsRemote = - RemoteDatabaseSource>; +pub type AverageBlockRewardsRemote = RemoteDatabaseSource< + PullAllWithAndSort, +>; pub type AverageBlockRewardsRemoteString = MapToString; @@ -87,24 +89,26 @@ define_and_impl_resolution_properties!( pub type AverageBlockRewards = DirectVecLocalDbChartSource; +type AverageBlockRewardsS = StripExt; pub type AverageBlockRewardsWeekly = DirectVecLocalDbChartSource< MapToString< - AverageLowerResolution, NewBlockRewardsInt, Week>, + AverageLowerResolution, NewBlockRewardsInt, Week>, >, Batch30Weeks, WeeklyProperties, >; pub type AverageBlockRewardsMonthly = DirectVecLocalDbChartSource< MapToString< - AverageLowerResolution, NewBlockRewardsInt, Month>, + AverageLowerResolution, NewBlockRewardsInt, Month>, >, Batch36Months, MonthlyProperties, >; +type AverageBlockRewardsMonthlyS = StripExt; pub type AverageBlockRewardsYearly = DirectVecLocalDbChartSource< MapToString< AverageLowerResolution< - MapParseTo, + MapParseTo, NewBlockRewardsMonthlyInt, Year, >, diff --git a/stats/stats/src/charts/lines/average_block_size.rs b/stats/stats/src/charts/lines/average_block_size.rs index ef04fb15c..965b81645 100644 --- a/stats/stats/src/charts/lines/average_block_size.rs +++ b/stats/stats/src/charts/lines/average_block_size.rs @@ -1,10 +1,11 @@ use std::ops::Range; use crate::{ + charts::db_interaction::read::QueryAllBlockTimestampRange, data_source::{ kinds::{ data_manipulation::{ - map::{MapParseTo, MapToString}, + map::{MapParseTo, MapToString, StripExt}, resolutions::average::AverageLowerResolution, }, local_db::{ @@ -23,16 +24,16 @@ use crate::{ ChartProperties, Named, }; -use chrono::NaiveDate; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{prelude::*, DbBackend, Statement}; +use sea_orm::{DbBackend, Statement}; use super::new_blocks::{NewBlocksInt, NewBlocksMonthlyInt}; pub struct AverageBlockSizeStatement; impl StatementFromRange for AverageBlockSizeStatement { - fn get_statement(range: Option>, _: &BlockscoutMigrations) -> Statement { + fn get_statement(range: Option>>, _: &BlockscoutMigrations) -> Statement { sql_with_range_filter_opt!( DbBackend::Postgres, r#" @@ -52,8 +53,9 @@ impl StatementFromRange for AverageBlockSizeStatement { } } -pub type AverageBlockSizeRemote = - RemoteDatabaseSource>; +pub type AverageBlockSizeRemote = RemoteDatabaseSource< + PullAllWithAndSort, +>; pub struct Properties; @@ -82,20 +84,26 @@ define_and_impl_resolution_properties!( pub type AverageBlockSize = DirectVecLocalDbChartSource; +type AverageBlockSizeS = StripExt; pub type AverageBlockSizeWeekly = DirectVecLocalDbChartSource< - MapToString, NewBlocksInt, Week>>, + MapToString, NewBlocksInt, Week>>, Batch30Weeks, WeeklyProperties, >; pub type AverageBlockSizeMonthly = DirectVecLocalDbChartSource< - MapToString, NewBlocksInt, Month>>, + MapToString, NewBlocksInt, Month>>, Batch36Months, MonthlyProperties, >; +type AverageBlockSizeMonthlyS = StripExt; pub type AverageBlockSizeYearly = DirectVecLocalDbChartSource< MapToString< - AverageLowerResolution, NewBlocksMonthlyInt, Year>, + AverageLowerResolution< + MapParseTo, + NewBlocksMonthlyInt, + Year, + >, >, Batch30Years, YearlyProperties, diff --git a/stats/stats/src/charts/lines/average_gas_limit.rs b/stats/stats/src/charts/lines/average_gas_limit.rs index 53111726b..81385b354 100644 --- a/stats/stats/src/charts/lines/average_gas_limit.rs +++ b/stats/stats/src/charts/lines/average_gas_limit.rs @@ -1,10 +1,11 @@ use std::ops::Range; use crate::{ + charts::db_interaction::read::QueryAllBlockTimestampRange, data_source::{ kinds::{ data_manipulation::{ - map::{MapParseTo, MapToString}, + map::{MapParseTo, MapToString, StripExt}, resolutions::average::AverageLowerResolution, }, local_db::{ @@ -23,16 +24,16 @@ use crate::{ ChartProperties, Named, }; -use chrono::NaiveDate; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{prelude::*, DbBackend, Statement}; +use sea_orm::{DbBackend, Statement}; use super::new_blocks::{NewBlocksInt, NewBlocksMonthlyInt}; pub struct AverageGasLimitStatement; impl StatementFromRange for AverageGasLimitStatement { - fn get_statement(range: Option>, _: &BlockscoutMigrations) -> Statement { + fn get_statement(range: Option>>, _: &BlockscoutMigrations) -> Statement { sql_with_range_filter_opt!( DbBackend::Postgres, r#" @@ -52,8 +53,9 @@ impl StatementFromRange for AverageGasLimitStatement { } } -pub type AverageGasLimitRemote = - RemoteDatabaseSource>; +pub type AverageGasLimitRemote = RemoteDatabaseSource< + PullAllWithAndSort, +>; pub struct Properties; @@ -82,19 +84,21 @@ define_and_impl_resolution_properties!( pub type AverageGasLimit = DirectVecLocalDbChartSource; +type AverageGasLimitS = StripExt; pub type AverageGasLimitWeekly = DirectVecLocalDbChartSource< - MapToString, NewBlocksInt, Week>>, + MapToString, NewBlocksInt, Week>>, Batch30Weeks, WeeklyProperties, >; pub type AverageGasLimitMonthly = DirectVecLocalDbChartSource< - MapToString, NewBlocksInt, Month>>, + MapToString, NewBlocksInt, Month>>, Batch36Months, MonthlyProperties, >; +type AverageGasLimitMonthlyS = StripExt; pub type AverageGasLimitYearly = DirectVecLocalDbChartSource< MapToString< - AverageLowerResolution, NewBlocksMonthlyInt, Year>, + AverageLowerResolution, NewBlocksMonthlyInt, Year>, >, Batch30Years, YearlyProperties, diff --git a/stats/stats/src/charts/lines/average_gas_price.rs b/stats/stats/src/charts/lines/average_gas_price.rs index f1a8b31ee..bf1c1c0c5 100644 --- a/stats/stats/src/charts/lines/average_gas_price.rs +++ b/stats/stats/src/charts/lines/average_gas_price.rs @@ -1,10 +1,11 @@ use std::ops::Range; use crate::{ + charts::db_interaction::read::QueryAllBlockTimestampRange, data_source::{ kinds::{ data_manipulation::{ - map::{MapParseTo, MapToString}, + map::{MapParseTo, MapToString, StripExt}, resolutions::average::AverageLowerResolution, }, local_db::{ @@ -23,9 +24,9 @@ use crate::{ ChartProperties, Named, }; -use chrono::NaiveDate; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{prelude::*, DbBackend, Statement}; +use sea_orm::{DbBackend, Statement}; use super::new_txns::{NewTxnsInt, NewTxnsMonthlyInt}; @@ -35,7 +36,7 @@ pub struct AverageGasPriceStatement; impl StatementFromRange for AverageGasPriceStatement { fn get_statement( - range: Option>, + range: Option>>, completed_migrations: &BlockscoutMigrations, ) -> Statement { if completed_migrations.denormalization { @@ -105,8 +106,9 @@ impl StatementFromRange for AverageGasPriceStatement { } } -pub type AverageGasPriceRemote = - RemoteDatabaseSource>; +pub type AverageGasPriceRemote = RemoteDatabaseSource< + PullAllWithAndSort, +>; pub type AverageGasPriceRemoteString = MapToString; @@ -137,19 +139,21 @@ define_and_impl_resolution_properties!( pub type AverageGasPrice = DirectVecLocalDbChartSource; +type AverageGasPriceS = StripExt; pub type AverageGasPriceWeekly = DirectVecLocalDbChartSource< - MapToString, NewTxnsInt, Week>>, + MapToString, NewTxnsInt, Week>>, Batch30Weeks, WeeklyProperties, >; pub type AverageGasPriceMonthly = DirectVecLocalDbChartSource< - MapToString, NewTxnsInt, Month>>, + MapToString, NewTxnsInt, Month>>, Batch36Months, MonthlyProperties, >; +type AverageGasPriceMonthlyS = StripExt; pub type AverageGasPriceYearly = DirectVecLocalDbChartSource< MapToString< - AverageLowerResolution, NewTxnsMonthlyInt, Year>, + AverageLowerResolution, NewTxnsMonthlyInt, Year>, >, Batch30Years, YearlyProperties, diff --git a/stats/stats/src/charts/lines/average_txn_fee.rs b/stats/stats/src/charts/lines/average_txn_fee.rs index 7d0d7b058..7470243a7 100644 --- a/stats/stats/src/charts/lines/average_txn_fee.rs +++ b/stats/stats/src/charts/lines/average_txn_fee.rs @@ -3,10 +3,11 @@ use std::ops::Range; use crate::{ + charts::db_interaction::read::QueryAllBlockTimestampRange, data_source::{ kinds::{ data_manipulation::{ - map::{MapParseTo, MapToString}, + map::{MapParseTo, MapToString, StripExt}, resolutions::average::AverageLowerResolution, }, local_db::{ @@ -25,9 +26,9 @@ use crate::{ ChartProperties, Named, }; -use chrono::NaiveDate; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{prelude::*, DbBackend, Statement}; +use sea_orm::{DbBackend, Statement}; use super::new_txns::{NewTxnsInt, NewTxnsMonthlyInt}; @@ -37,7 +38,7 @@ pub struct AverageTxnFeeStatement; impl StatementFromRange for AverageTxnFeeStatement { fn get_statement( - range: Option>, + range: Option>>, completed_migrations: &BlockscoutMigrations, ) -> Statement { if completed_migrations.denormalization { @@ -108,8 +109,9 @@ impl StatementFromRange for AverageTxnFeeStatement { } } -pub type AverageTxnFeeRemote = - RemoteDatabaseSource>; +pub type AverageTxnFeeRemote = RemoteDatabaseSource< + PullAllWithAndSort, +>; pub type AverageTxnFeeRemoteString = MapToString; @@ -140,19 +142,21 @@ define_and_impl_resolution_properties!( pub type AverageTxnFee = DirectVecLocalDbChartSource; +type AverageTxnFeeS = StripExt; pub type AverageTxnFeeWeekly = DirectVecLocalDbChartSource< - MapToString, NewTxnsInt, Week>>, + MapToString, NewTxnsInt, Week>>, Batch30Weeks, WeeklyProperties, >; pub type AverageTxnFeeMonthly = DirectVecLocalDbChartSource< - MapToString, NewTxnsInt, Month>>, + MapToString, NewTxnsInt, Month>>, Batch36Months, MonthlyProperties, >; +type AverageTxnFeeMonthlyS = StripExt; pub type AverageTxnFeeYearly = DirectVecLocalDbChartSource< MapToString< - AverageLowerResolution, NewTxnsMonthlyInt, Year>, + AverageLowerResolution, NewTxnsMonthlyInt, Year>, >, Batch30Years, YearlyProperties, diff --git a/stats/stats/src/charts/lines/contracts_growth.rs b/stats/stats/src/charts/lines/contracts_growth.rs index 47c32b118..9a3703f32 100644 --- a/stats/stats/src/charts/lines/contracts_growth.rs +++ b/stats/stats/src/charts/lines/contracts_growth.rs @@ -1,6 +1,6 @@ use crate::{ data_source::kinds::{ - data_manipulation::resolutions::last_value::LastValueLowerResolution, + data_manipulation::{map::StripExt, resolutions::last_value::LastValueLowerResolution}, local_db::{ parameters::update::batching::parameters::{Batch30Weeks, Batch30Years, Batch36Months}, DailyCumulativeLocalDbChartSource, DirectVecLocalDbChartSource, @@ -44,18 +44,20 @@ define_and_impl_resolution_properties!( ); pub type ContractsGrowth = DailyCumulativeLocalDbChartSource; +type ContractsGrowthS = StripExt; pub type ContractsGrowthWeekly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch30Weeks, WeeklyProperties, >; pub type ContractsGrowthMonthly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch36Months, MonthlyProperties, >; +type ContractsGrowthMonthlyS = StripExt; pub type ContractsGrowthYearly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch30Years, YearlyProperties, >; diff --git a/stats/stats/src/charts/lines/gas_used_growth.rs b/stats/stats/src/charts/lines/gas_used_growth.rs index 8680f2a36..cdce9900d 100644 --- a/stats/stats/src/charts/lines/gas_used_growth.rs +++ b/stats/stats/src/charts/lines/gas_used_growth.rs @@ -1,10 +1,11 @@ use std::ops::Range; use crate::{ + charts::db_interaction::read::QueryAllBlockTimestampRange, data_source::{ kinds::{ data_manipulation::{ - map::{Map, MapFunction}, + map::{Map, MapFunction, StripExt}, resolutions::last_value::LastValueLowerResolution, }, local_db::{ @@ -20,17 +21,18 @@ use crate::{ define_and_impl_resolution_properties, types::timespans::{DateValue, Month, Week, Year}, utils::sql_with_range_filter_opt, - ChartProperties, MissingDatePolicy, Named, UpdateError, + ChartError, ChartProperties, MissingDatePolicy, Named, }; -use chrono::NaiveDate; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{prelude::*, DbBackend, Statement}; +use rust_decimal::Decimal; +use sea_orm::{DbBackend, Statement}; pub struct GasUsedPartialStatement; impl StatementFromRange for GasUsedPartialStatement { - fn get_statement(range: Option>, _: &BlockscoutMigrations) -> Statement { + fn get_statement(range: Option>>, _: &BlockscoutMigrations) -> Statement { sql_with_range_filter_opt!( DbBackend::Postgres, r#" @@ -51,14 +53,15 @@ impl StatementFromRange for GasUsedPartialStatement { } } -pub type GasUsedPartialRemote = - RemoteDatabaseSource>; +pub type GasUsedPartialRemote = RemoteDatabaseSource< + PullAllWithAndSort, +>; pub struct IncrementsFromPartialSum; impl MapFunction>> for IncrementsFromPartialSum { type Output = Vec>; - fn function(inner_data: Vec>) -> Result { + fn function(inner_data: Vec>) -> Result { Ok(inner_data .into_iter() .scan(Decimal::ZERO, |state, mut next| { @@ -102,18 +105,20 @@ define_and_impl_resolution_properties!( ); pub type GasUsedGrowth = DailyCumulativeLocalDbChartSource; +type GasUsedGrowthS = StripExt; pub type GasUsedGrowthWeekly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch30Weeks, WeeklyProperties, >; pub type GasUsedGrowthMonthly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch36Months, MonthlyProperties, >; +type GasUsedGrowthMonthlyS = StripExt; pub type GasUsedGrowthYearly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch30Years, YearlyProperties, >; diff --git a/stats/stats/src/charts/lines/mock.rs b/stats/stats/src/charts/lines/mock.rs index f4f7a611b..af59c0878 100644 --- a/stats/stats/src/charts/lines/mock.rs +++ b/stats/stats/src/charts/lines/mock.rs @@ -10,15 +10,18 @@ use crate::{ UpdateContext, }, missing_date::fit_into_range, + range::{Incrementable, UniversalRange}, types::{timespans::DateValue, Timespan, TimespanValue}, - ChartProperties, MissingDatePolicy, Named, UpdateError, + ChartError, ChartProperties, MissingDatePolicy, Named, }; use chrono::{DateTime, Duration, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; use rand::{distributions::uniform::SampleUniform, rngs::StdRng, Rng, SeedableRng}; -use sea_orm::prelude::*; -use std::{marker::PhantomData, ops::Range}; +use std::{ + marker::PhantomData, + ops::{Bound, Range}, +}; /// non-inclusive range fn generate_intervals(mut start: NaiveDate, end: NaiveDate) -> Vec { @@ -51,7 +54,7 @@ pub fn mocked_lines( pub fn mock_trim_lines( data: Vec>, query_time: DateTime, - query_range: Option>, + query_range: UniversalRange>, policy: MissingDatePolicy, ) -> Vec> where @@ -59,11 +62,16 @@ where V: Clone, { let date_range_start = query_range - .clone() - .map(|r| T::from_date(r.start.date_naive())); + .start + .map(|start| T::from_date(start.date_naive())); let mut date_range_end = query_time; - if let Some(r) = query_range { - date_range_end = date_range_end.min(r.end) + let query_range_end_exclusive = match query_range.end { + Bound::Included(end) => Some(end.saturating_inc()), + Bound::Excluded(end) => Some(end), + Bound::Unbounded => None, + }; + if let Some(end_exclusive) = query_range_end_exclusive { + date_range_end = date_range_end.min(end_exclusive) } fit_into_range( data, @@ -91,8 +99,8 @@ where async fn query_data( cx: &UpdateContext<'_>, - range: Option>, - ) -> Result>, UpdateError> { + range: UniversalRange>, + ) -> Result>, ChartError> { let full_data = mocked_lines(DateRange::get(), ValueRange::get()); Ok(mock_trim_lines(full_data, cx.time, range, Policy::get())) } @@ -137,8 +145,8 @@ where async fn query_data( cx: &UpdateContext<'_>, - range: Option>, - ) -> Result { + range: UniversalRange>, + ) -> Result { Ok(mock_trim_lines(Data::get(), cx.time, range, Policy::get())) } } diff --git a/stats/stats/src/charts/lines/native_coin_holders_growth.rs b/stats/stats/src/charts/lines/native_coin_holders_growth.rs index d7fcd4667..9e2fc8c4e 100644 --- a/stats/stats/src/charts/lines/native_coin_holders_growth.rs +++ b/stats/stats/src/charts/lines/native_coin_holders_growth.rs @@ -5,7 +5,8 @@ use crate::{ data_source::{ kinds::{ data_manipulation::{ - map::MapParseTo, resolutions::last_value::LastValueLowerResolution, + map::{MapParseTo, StripExt}, + resolutions::last_value::LastValueLowerResolution, }, local_db::{ parameter_traits::{CreateBehaviour, UpdateBehaviour}, @@ -20,7 +21,7 @@ use crate::{ }, define_and_impl_resolution_properties, types::timespans::{DateValue, Month, Week, Year}, - ChartProperties, MissingDatePolicy, Named, UpdateError, + ChartError, ChartProperties, MissingDatePolicy, Named, }; use blockscout_db::entity::address_coin_balances_daily; @@ -95,7 +96,7 @@ impl UpdateBehaviour<(), (), NaiveDate> for Update { last_accurate_point: Option>, min_blockscout_block: i64, dependency_data_fetch_timer: &mut AggregateTimer, - ) -> Result<(), UpdateError> { + ) -> Result<(), ChartError> { update_sequentially_with_support_table( cx, chart_id, @@ -114,21 +115,23 @@ pub async fn update_sequentially_with_support_table( last_accurate_point: Option>, min_blockscout_block: i64, remote_fetch_timer: &mut AggregateTimer, -) -> Result<(), UpdateError> { +) -> Result<(), ChartError> { tracing::info!(chart =% Properties::key(), "start sequential update"); let all_days = match last_accurate_point { - Some(last_row) => { - get_unique_ordered_days(cx.blockscout, Some(last_row.timespan), remote_fetch_timer) - .await - .map_err(UpdateError::BlockscoutDB)? - } + Some(last_row) => get_unique_ordered_days( + cx.blockscout.connection.as_ref(), + Some(last_row.timespan), + remote_fetch_timer, + ) + .await + .map_err(ChartError::BlockscoutDB)?, None => { - clear_support_table(cx.db) + clear_support_table(cx.db.connection.as_ref()) .await - .map_err(UpdateError::BlockscoutDB)?; - get_unique_ordered_days(cx.blockscout, None, remote_fetch_timer) + .map_err(ChartError::BlockscoutDB)?; + get_unique_ordered_days(cx.blockscout.connection.as_ref(), None, remote_fetch_timer) .await - .map_err(UpdateError::BlockscoutDB)? + .map_err(ChartError::BlockscoutDB)? } }; @@ -143,18 +146,26 @@ pub async fn update_sequentially_with_support_table( ); // NOTE: we update support table and chart data in one transaction // to support invariant that support table has information about last day in chart data - let db_tx = cx.db.begin().await.map_err(UpdateError::StatsDB)?; - let data: Vec = - calculate_days_using_support_table(&db_tx, cx.blockscout, days.iter().copied()) - .await - .map_err(|e| UpdateError::Internal(e.to_string()))? - .into_iter() - .map(|result| result.active_model(chart_id, Some(min_blockscout_block))) - .collect(); + let db_tx = cx + .db + .connection + .begin() + .await + .map_err(ChartError::StatsDB)?; + let data: Vec = calculate_days_using_support_table( + &db_tx, + cx.blockscout.connection.as_ref(), + days.iter().copied(), + ) + .await + .map_err(|e| ChartError::Internal(e.to_string()))? + .into_iter() + .map(|result| result.active_model(chart_id, Some(min_blockscout_block))) + .collect(); insert_data_many(&db_tx, data) .await - .map_err(UpdateError::StatsDB)?; - db_tx.commit().await.map_err(UpdateError::StatsDB)?; + .map_err(ChartError::StatsDB)?; + db_tx.commit().await.map_err(ChartError::StatsDB)?; } Ok(()) } @@ -163,7 +174,7 @@ async fn calculate_days_using_support_table( db: &C1, blockscout: &C2, days: impl IntoIterator, -) -> Result>, UpdateError> +) -> Result>, ChartError> where C1: ConnectionTrait, C2: ConnectionTrait, @@ -171,7 +182,7 @@ where let mut result = vec![]; let new_holders_by_date = get_holder_changes_by_date(blockscout, days) .await - .map_err(|e| UpdateError::Internal(format!("cannot get new holders: {e}")))?; + .map_err(|e| ChartError::Internal(format!("cannot get new holders: {e}")))?; for (date, holders) in new_holders_by_date { // this check shouldnt be triggered if data in blockscout is correct, @@ -179,7 +190,7 @@ where let addresses = holders.iter().map(|h| &h.address).collect::>(); if addresses.len() != holders.len() { tracing::error!(addresses = ?addresses, date = ?date, "duplicate addresses in holders"); - return Err(UpdateError::Internal( + return Err(ChartError::Internal( "duplicate addresses in holders".to_string(), )); }; @@ -192,10 +203,10 @@ where update_current_holders(db, holders) .await - .map_err(|e| UpdateError::Internal(format!("cannot update holders: {e}")))?; + .map_err(|e| ChartError::Internal(format!("cannot update holders: {e}")))?; let new_count = count_current_holders(db) .await - .map_err(|e| UpdateError::Internal(format!("cannot count holders: {e}")))?; + .map_err(|e| ChartError::Internal(format!("cannot count holders: {e}")))?; result.push(DateValue:: { timespan: date, value: new_count.to_string(), @@ -384,19 +395,21 @@ define_and_impl_resolution_properties!( pub type NativeCoinHoldersGrowth = LocalDbChartSource<(), (), Create, Update, DefaultQueryVec, Properties>; -pub type NativeCoinHoldersGrowthInt = MapParseTo; +pub type NativeCoinHoldersGrowthInt = MapParseTo, i64>; +type NativeCoinHoldersGrowthS = StripExt; pub type NativeCoinHoldersGrowthWeekly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch30Weeks, WeeklyProperties, >; pub type NativeCoinHoldersGrowthMonthly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch36Months, MonthlyProperties, >; +type NativeCoinHoldersGrowthMonthlyS = StripExt; pub type NativeCoinHoldersGrowthYearly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch30Years, YearlyProperties, >; diff --git a/stats/stats/src/charts/lines/native_coin_supply.rs b/stats/stats/src/charts/lines/native_coin_supply.rs index 8b86f6a6e..269259dce 100644 --- a/stats/stats/src/charts/lines/native_coin_supply.rs +++ b/stats/stats/src/charts/lines/native_coin_supply.rs @@ -1,10 +1,12 @@ use std::ops::Range; use crate::{ + charts::db_interaction::read::QueryAllBlockTimestampRange, data_source::{ kinds::{ data_manipulation::{ - map::MapToString, resolutions::last_value::LastValueLowerResolution, + map::{MapToString, StripExt}, + resolutions::last_value::LastValueLowerResolution, }, local_db::{ parameters::update::batching::parameters::{ @@ -21,16 +23,16 @@ use crate::{ ChartProperties, Named, }; -use chrono::NaiveDate; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{prelude::*, DbBackend, Statement}; +use sea_orm::{DbBackend, Statement}; const ETH: i64 = 1_000_000_000_000_000_000; pub struct NativeCoinSupplyStatement; impl StatementFromRange for NativeCoinSupplyStatement { - fn get_statement(range: Option>, _: &BlockscoutMigrations) -> Statement { + fn get_statement(range: Option>>, _: &BlockscoutMigrations) -> Statement { let day_range: Option> = range.map(|r| { let Range { start, end } = r; // chart is off anyway, so shouldn't be a big deal @@ -88,8 +90,9 @@ impl StatementFromRange for NativeCoinSupplyStatement { } // query returns float value -pub type NativeCoinSupplyRemote = - RemoteDatabaseSource>; +pub type NativeCoinSupplyRemote = RemoteDatabaseSource< + PullAllWithAndSort, +>; pub type NativeCoinSupplyRemoteString = MapToString; @@ -120,18 +123,20 @@ define_and_impl_resolution_properties!( pub type NativeCoinSupply = DirectVecLocalDbChartSource; +type NativeCoinSupplyS = StripExt; pub type NativeCoinSupplyWeekly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch30Weeks, WeeklyProperties, >; pub type NativeCoinSupplyMonthly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch36Months, MonthlyProperties, >; +type NativeCoinSupplyMonthlyS = StripExt; pub type NativeCoinSupplyYearly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch30Years, YearlyProperties, >; diff --git a/stats/stats/src/charts/lines/new_accounts.rs b/stats/stats/src/charts/lines/new_accounts.rs index 0644a955c..24f5da7a6 100644 --- a/stats/stats/src/charts/lines/new_accounts.rs +++ b/stats/stats/src/charts/lines/new_accounts.rs @@ -1,11 +1,11 @@ use std::ops::Range; use crate::{ - charts::types::timespans::DateValue, + charts::{db_interaction::read::QueryAllBlockTimestampRange, types::timespans::DateValue}, data_source::{ kinds::{ data_manipulation::{ - map::{MapParseTo, MapToString}, + map::{MapParseTo, MapToString, StripExt}, resolutions::sum::SumLowerResolution, }, local_db::{ @@ -21,24 +21,25 @@ use crate::{ }, define_and_impl_resolution_properties, missing_date::trim_out_of_range_sorted, + range::{data_source_query_range_to_db_statement_range, UniversalRange}, types::timespans::{Month, Week, Year}, utils::sql_with_range_filter_opt, - ChartProperties, Named, UpdateError, + ChartError, ChartProperties, Named, }; -use chrono::NaiveDate; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{prelude::*, DbBackend, FromQueryResult, Statement}; +use sea_orm::{DbBackend, FromQueryResult, Statement}; pub struct NewAccountsStatement; impl StatementFromRange for NewAccountsStatement { fn get_statement( - range: Option>, + range: Option>>, completed_migrations: &BlockscoutMigrations, ) -> Statement { // `MIN_UTC` does not fit into postgres' timestamp. Unix epoch start should be enough - let min_timestamp = DateTimeUtc::UNIX_EPOCH; + let min_timestamp = DateTime::::UNIX_EPOCH; // All transactions from the beginning must be considered to calculate new accounts correctly. // E.g. if account was first active both before `range.start()` and within the range, // we don't want to count it within the range (as it's not a *new* account). @@ -99,17 +100,22 @@ impl RemoteQueryBehaviour for NewAccountsQueryBehaviour { async fn query_data( cx: &UpdateContext<'_>, - range: Option>, - ) -> Result>, UpdateError> { - let query = - NewAccountsStatement::get_statement(range.clone(), &cx.blockscout_applied_migrations); + range: UniversalRange>, + ) -> Result>, ChartError> { + let statement_range = + data_source_query_range_to_db_statement_range::(cx, range) + .await?; + let query = NewAccountsStatement::get_statement( + statement_range.clone(), + &cx.blockscout_applied_migrations, + ); let mut data = DateValue::::find_by_statement(query) - .all(cx.blockscout) + .all(cx.blockscout.connection.as_ref()) .await - .map_err(UpdateError::BlockscoutDB)?; + .map_err(ChartError::BlockscoutDB)?; // make sure that it's sorted data.sort_by_key(|d| d.timespan); - if let Some(range) = range { + if let Some(range) = statement_range { let range = range.start.date_naive()..=range.end.date_naive(); trim_out_of_range_sorted(&mut data, range); } @@ -150,7 +156,7 @@ define_and_impl_resolution_properties!( ); pub type NewAccounts = DirectVecLocalDbChartSource; -pub type NewAccountsInt = MapParseTo; +pub type NewAccountsInt = MapParseTo, i64>; pub type NewAccountsWeekly = DirectVecLocalDbChartSource< MapToString>, Batch30Weeks, @@ -161,7 +167,7 @@ pub type NewAccountsMonthly = DirectVecLocalDbChartSource< Batch36Months, MonthlyProperties, >; -pub type NewAccountsMonthlyInt = MapParseTo; +pub type NewAccountsMonthlyInt = MapParseTo, i64>; pub type NewAccountsYearly = DirectVecLocalDbChartSource< MapToString>, Batch30Years, diff --git a/stats/stats/src/charts/lines/new_block_rewards.rs b/stats/stats/src/charts/lines/new_block_rewards.rs index f3adf5755..225610c99 100644 --- a/stats/stats/src/charts/lines/new_block_rewards.rs +++ b/stats/stats/src/charts/lines/new_block_rewards.rs @@ -6,9 +6,13 @@ use std::ops::Range; use crate::{ + charts::db_interaction::read::QueryAllBlockTimestampRange, data_source::{ kinds::{ - data_manipulation::{map::MapParseTo, resolutions::sum::SumLowerResolution}, + data_manipulation::{ + map::{MapParseTo, StripExt}, + resolutions::sum::SumLowerResolution, + }, local_db::{ parameters::update::batching::parameters::Batch30Days, DirectVecLocalDbChartSource, }, @@ -22,14 +26,14 @@ use crate::{ ChartProperties, Named, }; -use chrono::NaiveDate; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{prelude::*, DbBackend, Statement}; +use sea_orm::{DbBackend, Statement}; pub struct NewBlockRewardsStatement; impl StatementFromRange for NewBlockRewardsStatement { - fn get_statement(range: Option>, _: &BlockscoutMigrations) -> Statement { + fn get_statement(range: Option>>, _: &BlockscoutMigrations) -> Statement { sql_with_range_filter_opt!( DbBackend::Postgres, r#" @@ -50,8 +54,9 @@ impl StatementFromRange for NewBlockRewardsStatement { } } -pub type NewBlockRewardsRemote = - RemoteDatabaseSource>; +pub type NewBlockRewardsRemote = RemoteDatabaseSource< + PullAllWithAndSort, +>; pub struct Properties; @@ -80,7 +85,7 @@ define_and_impl_resolution_properties!( pub type NewBlockRewards = DirectVecLocalDbChartSource; -pub type NewBlockRewardsInt = MapParseTo; +pub type NewBlockRewardsInt = MapParseTo, i64>; pub type NewBlockRewardsMonthlyInt = SumLowerResolution; #[cfg(test)] @@ -93,8 +98,9 @@ mod tests { use super::*; use crate::{ data_source::{types::BlockscoutMigrations, DataSource, UpdateContext, UpdateParameters}, + range::UniversalRange, tests::{ - init_db::init_db_all, + init_db::init_marked_db_all, mock_blockscout::fill_mock_blockscout_data, simple_test::{map_str_tuple_to_owned, simple_test_chart}, }, @@ -131,13 +137,13 @@ mod tests { ("2023-02-01", "1"), ("2023-03-01", "1"), ]); - let (db, blockscout) = init_db_all("update_new_block_rewards_monthly_int").await; + let (db, blockscout) = init_marked_db_all("update_new_block_rewards_monthly_int").await; let current_time = chrono::DateTime::from_str("2023-03-01T12:00:00Z").unwrap(); let current_date = current_time.date_naive(); - NewBlockRewardsMonthlyInt::init_recursively(&db, ¤t_time) + NewBlockRewardsMonthlyInt::init_recursively(&db.connection, ¤t_time) .await .unwrap(); - fill_mock_blockscout_data(&blockscout, current_date).await; + fill_mock_blockscout_data(&blockscout.connection, current_date).await; let parameters = UpdateParameters { db: &db, @@ -151,12 +157,13 @@ mod tests { .await .unwrap(); let mut timer = AggregateTimer::new(); - let data: Vec<_> = NewBlockRewardsMonthlyInt::query_data(&cx, None, &mut timer) - .await - .unwrap() - .into_iter() - .map(|p| (p.timespan.into_date().to_string(), p.value.to_string())) - .collect(); + let data: Vec<_> = + NewBlockRewardsMonthlyInt::query_data(&cx, UniversalRange::full(), &mut timer) + .await + .unwrap() + .into_iter() + .map(|p| (p.timespan.into_date().to_string(), p.value.to_string())) + .collect(); assert_eq!(data, expected); } } diff --git a/stats/stats/src/charts/lines/new_blocks.rs b/stats/stats/src/charts/lines/new_blocks.rs index b1d5b4c8c..edcc33045 100644 --- a/stats/stats/src/charts/lines/new_blocks.rs +++ b/stats/stats/src/charts/lines/new_blocks.rs @@ -1,10 +1,11 @@ use std::ops::Range; use crate::{ + charts::db_interaction::read::QueryAllBlockTimestampRange, data_source::{ kinds::{ data_manipulation::{ - map::{MapParseTo, MapToString}, + map::{MapParseTo, MapToString, StripExt}, resolutions::sum::SumLowerResolution, }, local_db::{ @@ -23,14 +24,14 @@ use crate::{ ChartProperties, Named, }; -use chrono::NaiveDate; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{prelude::*, DbBackend, Statement}; +use sea_orm::{DbBackend, Statement}; pub struct NewBlocksStatement; impl StatementFromRange for NewBlocksStatement { - fn get_statement(range: Option>, _: &BlockscoutMigrations) -> Statement { + fn get_statement(range: Option>>, _: &BlockscoutMigrations) -> Statement { sql_with_range_filter_opt!( DbBackend::Postgres, r#" @@ -50,8 +51,9 @@ impl StatementFromRange for NewBlocksStatement { } } -pub type NewBlocksRemote = - RemoteDatabaseSource>; +pub type NewBlocksRemote = RemoteDatabaseSource< + PullAllWithAndSort, +>; pub struct Properties; @@ -79,7 +81,7 @@ define_and_impl_resolution_properties!( ); pub type NewBlocks = DirectVecLocalDbChartSource; -pub type NewBlocksInt = MapParseTo; +pub type NewBlocksInt = MapParseTo, i64>; pub type NewBlocksWeekly = DirectVecLocalDbChartSource< MapToString>, Batch30Weeks, @@ -90,7 +92,7 @@ pub type NewBlocksMonthly = DirectVecLocalDbChartSource< Batch36Months, MonthlyProperties, >; -pub type NewBlocksMonthlyInt = MapParseTo; +pub type NewBlocksMonthlyInt = MapParseTo, i64>; pub type NewBlocksYearly = DirectVecLocalDbChartSource< MapToString>, Batch30Years, @@ -103,9 +105,10 @@ mod tests { use crate::{ charts::db_interaction::read::get_min_block_blockscout, data_source::{types::BlockscoutMigrations, DataSource, UpdateContext}, - get_line_chart_data, + query_dispatch::{serialize_line_points, QuerySerialized}, + range::UniversalRange, tests::{ - init_db::init_db_all, mock_blockscout::fill_mock_blockscout_data, + init_db::init_marked_db_all, mock_blockscout::fill_mock_blockscout_data, point_construction::dt, simple_test::simple_test_chart, }, types::ExtendedTimespanValue, @@ -114,23 +117,25 @@ mod tests { use chrono::{NaiveDate, Utc}; use entity::{chart_data, charts}; use pretty_assertions::assert_eq; - use sea_orm::{DatabaseConnection, EntityTrait, Set}; + use sea_orm::{EntityTrait, Set}; use std::str::FromStr; #[tokio::test] #[ignore = "needs database to run"] async fn update_new_blocks_recurrent() { let _ = tracing_subscriber::fmt::try_init(); - let (db, blockscout) = init_db_all("update_new_blocks_recurrent").await; + let (db, blockscout) = init_marked_db_all("update_new_blocks_recurrent").await; let current_time = chrono::DateTime::::from_str("2022-11-12T12:00:00Z").unwrap(); let current_date = current_time.date_naive(); - fill_mock_blockscout_data(&blockscout, current_date).await; + fill_mock_blockscout_data(blockscout.connection.as_ref(), current_date).await; - NewBlocks::init_recursively(&db, ¤t_time) + NewBlocks::init_recursively(db.connection.as_ref(), ¤t_time) .await .unwrap(); - let min_blockscout_block = get_min_block_blockscout(&blockscout).await.unwrap(); + let min_blockscout_block = get_min_block_blockscout(blockscout.connection.as_ref()) + .await + .unwrap(); // set wrong value and check, that it was rewritten chart_data::Entity::insert_many([ chart_data::ActiveModel { @@ -148,7 +153,7 @@ mod tests { ..Default::default() }, ]) - .exec(&db as &DatabaseConnection) + .exec(db.connection.as_ref()) .await .unwrap(); // set corresponding `last_updated_at` for successful partial update @@ -157,7 +162,7 @@ mod tests { last_updated_at: Set(Some(dt("2022-11-12T11:00:00").and_utc().fixed_offset())), ..Default::default() }) - .exec(&db as &DatabaseConnection) + .exec(db.connection.as_ref()) .await .unwrap(); @@ -171,18 +176,9 @@ mod tests { force_full: false, }; NewBlocks::update_recursively(&cx).await.unwrap(); - let data = get_line_chart_data::( - &db, - &NewBlocks::name(), - None, - None, - None, - crate::MissingDatePolicy::FillZero, - false, - 1, - ) - .await - .unwrap(); + let data = NewBlocks::query_data_static(&cx, UniversalRange::full(), None, false) + .await + .unwrap(); let expected = vec![ ExtendedTimespanValue { timespan: NaiveDate::from_str("2022-11-10").unwrap(), @@ -200,25 +196,16 @@ mod tests { is_approximate: true, }, ]; - assert_eq!(expected, data); + assert_eq!(serialize_line_points(expected), data); // note that update is full, therefore there is entry with date `2022-11-09` cx.force_full = true; // need to update time so that the update is not ignored as the same one cx.time = chrono::DateTime::::from_str("2022-11-12T13:00:00Z").unwrap(); NewBlocks::update_recursively(&cx).await.unwrap(); - let data = get_line_chart_data::( - &db, - &NewBlocks::name(), - None, - None, - None, - crate::MissingDatePolicy::FillZero, - false, - 1, - ) - .await - .unwrap(); + let data = NewBlocks::query_data_static(&cx, UniversalRange::full(), None, false) + .await + .unwrap(); let expected = vec![ ExtendedTimespanValue { timespan: NaiveDate::from_str("2022-11-09").unwrap(), @@ -241,19 +228,19 @@ mod tests { is_approximate: true, }, ]; - assert_eq!(expected, data); + assert_eq!(serialize_line_points(expected), data); } #[tokio::test] #[ignore = "needs database to run"] async fn update_new_blocks_fresh() { let _ = tracing_subscriber::fmt::try_init(); - let (db, blockscout) = init_db_all("update_new_blocks_fresh").await; + let (db, blockscout) = init_marked_db_all("update_new_blocks_fresh").await; let current_time = chrono::DateTime::from_str("2022-11-12T12:00:00Z").unwrap(); let current_date = current_time.date_naive(); - fill_mock_blockscout_data(&blockscout, current_date).await; + fill_mock_blockscout_data(blockscout.connection.as_ref(), current_date).await; - NewBlocks::init_recursively(&db, ¤t_time) + NewBlocks::init_recursively(db.connection.as_ref(), ¤t_time) .await .unwrap(); @@ -265,18 +252,9 @@ mod tests { force_full: true, }; NewBlocks::update_recursively(&cx).await.unwrap(); - let data = get_line_chart_data::( - &db, - &NewBlocks::name(), - None, - None, - None, - crate::MissingDatePolicy::FillZero, - false, - 0, - ) - .await - .unwrap(); + let data = NewBlocks::query_data_static(&cx, UniversalRange::full(), None, false) + .await + .unwrap(); let expected = vec![ ExtendedTimespanValue { timespan: NaiveDate::from_str("2022-11-09").unwrap(), @@ -296,26 +274,28 @@ mod tests { ExtendedTimespanValue { timespan: NaiveDate::from_str("2022-11-12").unwrap(), value: "1".into(), - is_approximate: false, + is_approximate: true, }, ]; - assert_eq!(expected, data); + assert_eq!(serialize_line_points(expected), data); } #[tokio::test] #[ignore = "needs database to run"] async fn update_new_blocks_last() { let _ = tracing_subscriber::fmt::try_init(); - let (db, blockscout) = init_db_all("update_new_blocks_last").await; + let (db, blockscout) = init_marked_db_all("update_new_blocks_last").await; let current_time = chrono::DateTime::from_str("2022-11-12T12:00:00Z").unwrap(); let current_date = current_time.date_naive(); - fill_mock_blockscout_data(&blockscout, current_date).await; + fill_mock_blockscout_data(blockscout.connection.as_ref(), current_date).await; - NewBlocks::init_recursively(&db, ¤t_time) + NewBlocks::init_recursively(db.connection.as_ref(), ¤t_time) .await .unwrap(); - let min_blockscout_block = get_min_block_blockscout(&blockscout).await.unwrap(); + let min_blockscout_block = get_min_block_blockscout(blockscout.connection.as_ref()) + .await + .unwrap(); // set wrong values and check, that they weren't rewritten // except the last one chart_data::Entity::insert_many([ @@ -348,7 +328,7 @@ mod tests { ..Default::default() }, ]) - .exec(&db as &DatabaseConnection) + .exec(db.connection.as_ref()) .await .unwrap(); // set corresponding `last_updated_at` for successful partial update @@ -357,7 +337,7 @@ mod tests { last_updated_at: Set(Some(dt("2022-11-12T11:00:00").and_utc().fixed_offset())), ..Default::default() }) - .exec(&db as &DatabaseConnection) + .exec(db.connection.as_ref()) .await .unwrap(); @@ -369,18 +349,9 @@ mod tests { force_full: false, }; NewBlocks::update_recursively(&cx).await.unwrap(); - let data = get_line_chart_data::( - &db, - &NewBlocks::name(), - None, - None, - None, - crate::MissingDatePolicy::FillZero, - false, - 1, - ) - .await - .unwrap(); + let data = NewBlocks::query_data_static(&cx, UniversalRange::full(), None, false) + .await + .unwrap(); let expected = vec![ ExtendedTimespanValue { timespan: NaiveDate::from_str("2022-11-09").unwrap(), @@ -403,7 +374,7 @@ mod tests { is_approximate: true, }, ]; - assert_eq!(expected, data); + assert_eq!(serialize_line_points(expected), data); } #[tokio::test] diff --git a/stats/stats/src/charts/lines/new_contracts.rs b/stats/stats/src/charts/lines/new_contracts.rs index 3f91a7727..b6702b6be 100644 --- a/stats/stats/src/charts/lines/new_contracts.rs +++ b/stats/stats/src/charts/lines/new_contracts.rs @@ -1,10 +1,11 @@ use std::ops::Range; use crate::{ + charts::db_interaction::read::QueryAllBlockTimestampRange, data_source::{ kinds::{ data_manipulation::{ - map::{MapParseTo, MapToString}, + map::{MapParseTo, MapToString, StripExt}, resolutions::sum::SumLowerResolution, }, local_db::{ @@ -23,15 +24,15 @@ use crate::{ ChartProperties, Named, }; -use chrono::NaiveDate; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{prelude::DateTimeUtc, DbBackend, Statement}; +use sea_orm::{DbBackend, Statement}; pub struct NewContractsStatement; impl StatementFromRange for NewContractsStatement { fn get_statement( - range: Option>, + range: Option>>, completed_migrations: &BlockscoutMigrations, ) -> Statement { if completed_migrations.denormalization { @@ -116,8 +117,9 @@ impl StatementFromRange for NewContractsStatement { } } -pub type NewContractsRemote = - RemoteDatabaseSource>; +pub type NewContractsRemote = RemoteDatabaseSource< + PullAllWithAndSort, +>; pub struct Properties; @@ -145,7 +147,7 @@ define_and_impl_resolution_properties!( ); pub type NewContracts = DirectVecLocalDbChartSource; -pub type NewContractsInt = MapParseTo; +pub type NewContractsInt = MapParseTo, i64>; pub type NewContractsWeekly = DirectVecLocalDbChartSource< MapToString>, Batch30Weeks, @@ -156,7 +158,7 @@ pub type NewContractsMonthly = DirectVecLocalDbChartSource< Batch36Months, MonthlyProperties, >; -pub type NewContractsMonthlyInt = MapParseTo; +pub type NewContractsMonthlyInt = MapParseTo, i64>; pub type NewContractsYearly = DirectVecLocalDbChartSource< MapToString>, Batch30Years, diff --git a/stats/stats/src/charts/lines/new_native_coin_holders.rs b/stats/stats/src/charts/lines/new_native_coin_holders.rs index 797942bdd..d764e071a 100644 --- a/stats/stats/src/charts/lines/new_native_coin_holders.rs +++ b/stats/stats/src/charts/lines/new_native_coin_holders.rs @@ -3,7 +3,7 @@ use crate::{ data_source::kinds::{ data_manipulation::{ delta::Delta, - map::{MapParseTo, MapToString}, + map::{MapParseTo, MapToString, StripExt}, resolutions::sum::SumLowerResolution, }, local_db::{ @@ -52,7 +52,7 @@ pub type NewNativeCoinHolders = DirectVecLocalDbChartSource< Batch30Days, Properties, >; -pub type NewNativeCoinHoldersInt = MapParseTo; +pub type NewNativeCoinHoldersInt = MapParseTo, i64>; pub type NewNativeCoinHoldersWeekly = DirectVecLocalDbChartSource< MapToString>, Batch30Weeks, @@ -63,7 +63,7 @@ pub type NewNativeCoinHoldersMonthly = DirectVecLocalDbChartSource< Batch36Months, MonthlyProperties, >; -pub type NewNativeCoinHoldersMonthlyInt = MapParseTo; +pub type NewNativeCoinHoldersMonthlyInt = MapParseTo, i64>; pub type NewNativeCoinHoldersYearly = DirectVecLocalDbChartSource< MapToString>, Batch30Years, diff --git a/stats/stats/src/charts/lines/new_native_coin_transfers.rs b/stats/stats/src/charts/lines/new_native_coin_transfers.rs index 7612143ed..53fcc1cac 100644 --- a/stats/stats/src/charts/lines/new_native_coin_transfers.rs +++ b/stats/stats/src/charts/lines/new_native_coin_transfers.rs @@ -1,10 +1,11 @@ use std::ops::Range; use crate::{ + charts::db_interaction::read::QueryAllBlockTimestampRange, data_source::{ kinds::{ data_manipulation::{ - map::{MapParseTo, MapToString}, + map::{MapParseTo, MapToString, StripExt}, resolutions::sum::SumLowerResolution, }, local_db::{ @@ -23,15 +24,15 @@ use crate::{ ChartProperties, Named, }; -use chrono::NaiveDate; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{prelude::*, DbBackend, Statement}; +use sea_orm::{DbBackend, Statement}; pub struct NewNativeCoinTransfersStatement; impl StatementFromRange for NewNativeCoinTransfersStatement { fn get_statement( - range: Option>, + range: Option>>, completed_migrations: &BlockscoutMigrations, ) -> Statement { if completed_migrations.denormalization { @@ -77,8 +78,14 @@ impl StatementFromRange for NewNativeCoinTransfersStatement { } } -pub type NewNativeCoinTransfersRemote = - RemoteDatabaseSource>; +pub type NewNativeCoinTransfersRemote = RemoteDatabaseSource< + PullAllWithAndSort< + NewNativeCoinTransfersStatement, + NaiveDate, + String, + QueryAllBlockTimestampRange, + >, +>; pub struct Properties; @@ -107,7 +114,7 @@ define_and_impl_resolution_properties!( pub type NewNativeCoinTransfers = DirectVecLocalDbChartSource; -pub type NewNativeCoinTransfersInt = MapParseTo; +pub type NewNativeCoinTransfersInt = MapParseTo, i64>; pub type NewNativeCoinTransfersWeekly = DirectVecLocalDbChartSource< MapToString>, Batch30Weeks, @@ -118,7 +125,8 @@ pub type NewNativeCoinTransfersMonthly = DirectVecLocalDbChartSource< Batch36Months, MonthlyProperties, >; -pub type NewNativeCoinTransfersMonthlyInt = MapParseTo; +pub type NewNativeCoinTransfersMonthlyInt = + MapParseTo, i64>; pub type NewNativeCoinTransfersYearly = DirectVecLocalDbChartSource< MapToString>, Batch30Years, diff --git a/stats/stats/src/charts/lines/new_txns.rs b/stats/stats/src/charts/lines/new_txns.rs index 76a6029bb..2f3b3fd26 100644 --- a/stats/stats/src/charts/lines/new_txns.rs +++ b/stats/stats/src/charts/lines/new_txns.rs @@ -1,10 +1,11 @@ use std::ops::Range; use crate::{ + charts::db_interaction::read::QueryAllBlockTimestampRange, data_source::{ kinds::{ data_manipulation::{ - map::{MapParseTo, MapToString}, + map::{MapParseTo, MapToString, StripExt}, resolutions::sum::SumLowerResolution, }, local_db::{ @@ -23,15 +24,15 @@ use crate::{ ChartProperties, Named, }; -use chrono::NaiveDate; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{prelude::*, DbBackend, Statement}; +use sea_orm::{DbBackend, Statement}; pub struct NewTxnsStatement; impl StatementFromRange for NewTxnsStatement { fn get_statement( - range: Option>, + range: Option>>, completed_migrations: &BlockscoutMigrations, ) -> Statement { if completed_migrations.denormalization { @@ -73,8 +74,9 @@ impl StatementFromRange for NewTxnsStatement { } } -pub type NewTxnsRemote = - RemoteDatabaseSource>; +pub type NewTxnsRemote = RemoteDatabaseSource< + PullAllWithAndSort, +>; pub struct Properties; @@ -102,7 +104,7 @@ define_and_impl_resolution_properties!( ); pub type NewTxns = DirectVecLocalDbChartSource; -pub type NewTxnsInt = MapParseTo; +pub type NewTxnsInt = MapParseTo, i64>; pub type NewTxnsWeekly = DirectVecLocalDbChartSource< MapToString>, Batch30Weeks, @@ -113,7 +115,7 @@ pub type NewTxnsMonthly = DirectVecLocalDbChartSource< Batch36Months, MonthlyProperties, >; -pub type NewTxnsMonthlyInt = MapParseTo; +pub type NewTxnsMonthlyInt = MapParseTo, i64>; pub type NewTxnsYearly = DirectVecLocalDbChartSource< MapToString>, Batch30Years, diff --git a/stats/stats/src/charts/lines/new_verified_contracts.rs b/stats/stats/src/charts/lines/new_verified_contracts.rs index e4a0c1e6b..c448c0f0c 100644 --- a/stats/stats/src/charts/lines/new_verified_contracts.rs +++ b/stats/stats/src/charts/lines/new_verified_contracts.rs @@ -1,10 +1,11 @@ use std::ops::Range; use crate::{ + charts::db_interaction::read::QueryAllBlockTimestampRange, data_source::{ kinds::{ data_manipulation::{ - map::{MapParseTo, MapToString}, + map::{MapParseTo, MapToString, StripExt}, resolutions::sum::SumLowerResolution, }, local_db::{ @@ -23,14 +24,14 @@ use crate::{ ChartProperties, Named, }; -use chrono::NaiveDate; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{prelude::*, DbBackend, Statement}; +use sea_orm::{DbBackend, Statement}; pub struct NewVerifiedContractsStatement; impl StatementFromRange for NewVerifiedContractsStatement { - fn get_statement(range: Option>, _: &BlockscoutMigrations) -> Statement { + fn get_statement(range: Option>>, _: &BlockscoutMigrations) -> Statement { sql_with_range_filter_opt!( DbBackend::Postgres, r#" @@ -48,8 +49,14 @@ impl StatementFromRange for NewVerifiedContractsStatement { } } -pub type NewVerifiedContractsRemote = - RemoteDatabaseSource>; +pub type NewVerifiedContractsRemote = RemoteDatabaseSource< + PullAllWithAndSort< + NewVerifiedContractsStatement, + NaiveDate, + String, + QueryAllBlockTimestampRange, + >, +>; pub struct Properties; @@ -78,7 +85,7 @@ define_and_impl_resolution_properties!( pub type NewVerifiedContracts = DirectVecLocalDbChartSource; -pub type NewVerifiedContractsInt = MapParseTo; +pub type NewVerifiedContractsInt = MapParseTo, i64>; pub type NewVerifiedContractsWeekly = DirectVecLocalDbChartSource< MapToString>, Batch30Weeks, @@ -89,7 +96,7 @@ pub type NewVerifiedContractsMonthly = DirectVecLocalDbChartSource< Batch36Months, MonthlyProperties, >; -pub type NewVerifiedContractsMonthlyInt = MapParseTo; +pub type NewVerifiedContractsMonthlyInt = MapParseTo, i64>; pub type NewVerifiedContractsYearly = DirectVecLocalDbChartSource< MapToString>, Batch30Years, diff --git a/stats/stats/src/charts/lines/txns_fee.rs b/stats/stats/src/charts/lines/txns_fee.rs index 5a93a2362..18f363d02 100644 --- a/stats/stats/src/charts/lines/txns_fee.rs +++ b/stats/stats/src/charts/lines/txns_fee.rs @@ -3,10 +3,11 @@ use std::ops::Range; use crate::{ + charts::db_interaction::read::QueryAllBlockTimestampRange, data_source::{ kinds::{ data_manipulation::{ - map::{MapParseTo, MapToString}, + map::{MapParseTo, MapToString, StripExt}, resolutions::sum::SumLowerResolution, }, local_db::{ @@ -25,9 +26,9 @@ use crate::{ ChartProperties, Named, }; -use chrono::NaiveDate; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{prelude::*, DbBackend, Statement}; +use sea_orm::{DbBackend, Statement}; const ETHER: i64 = i64::pow(10, 18); @@ -35,7 +36,7 @@ pub struct TxnsFeeStatement; impl StatementFromRange for TxnsFeeStatement { fn get_statement( - range: Option>, + range: Option>>, completed_migrations: &BlockscoutMigrations, ) -> Statement { if completed_migrations.denormalization { @@ -106,7 +107,9 @@ impl StatementFromRange for TxnsFeeStatement { } } -pub type TxnsFeeRemote = RemoteDatabaseSource>; +pub type TxnsFeeRemote = RemoteDatabaseSource< + PullAllWithAndSort, +>; pub type TxnsFeeRemoteString = MapToString; @@ -136,7 +139,7 @@ define_and_impl_resolution_properties!( ); pub type TxnsFee = DirectVecLocalDbChartSource; -pub type TxnsFeeFloat = MapParseTo; +pub type TxnsFeeFloat = MapParseTo, f64>; pub type TxnsFeeWeekly = DirectVecLocalDbChartSource< MapToString>, Batch30Weeks, @@ -147,7 +150,7 @@ pub type TxnsFeeMonthly = DirectVecLocalDbChartSource< Batch36Months, MonthlyProperties, >; -pub type TxnsFeeMonthlyFloat = MapParseTo; +pub type TxnsFeeMonthlyFloat = MapParseTo, f64>; pub type TxnsFeeYearly = DirectVecLocalDbChartSource< MapToString>, Batch30Years, diff --git a/stats/stats/src/charts/lines/txns_growth.rs b/stats/stats/src/charts/lines/txns_growth.rs index bbffcab5d..344776fa8 100644 --- a/stats/stats/src/charts/lines/txns_growth.rs +++ b/stats/stats/src/charts/lines/txns_growth.rs @@ -1,7 +1,7 @@ use crate::{ charts::chart::ChartProperties, data_source::kinds::{ - data_manipulation::resolutions::last_value::LastValueLowerResolution, + data_manipulation::{map::StripExt, resolutions::last_value::LastValueLowerResolution}, local_db::{ parameters::update::batching::parameters::{Batch30Weeks, Batch30Years, Batch36Months}, DailyCumulativeLocalDbChartSource, DirectVecLocalDbChartSource, @@ -45,18 +45,20 @@ define_and_impl_resolution_properties!( ); pub type TxnsGrowth = DailyCumulativeLocalDbChartSource; +type TxnsGrowthS = StripExt; pub type TxnsGrowthWeekly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch30Weeks, WeeklyProperties, >; pub type TxnsGrowthMonthly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch36Months, MonthlyProperties, >; +type TxnsGrowthMonthlyS = StripExt; pub type TxnsGrowthYearly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch30Years, YearlyProperties, >; diff --git a/stats/stats/src/charts/lines/txns_success_rate.rs b/stats/stats/src/charts/lines/txns_success_rate.rs index 3bb543605..526b893d7 100644 --- a/stats/stats/src/charts/lines/txns_success_rate.rs +++ b/stats/stats/src/charts/lines/txns_success_rate.rs @@ -1,10 +1,11 @@ use std::ops::Range; use crate::{ + charts::db_interaction::read::QueryAllBlockTimestampRange, data_source::{ kinds::{ data_manipulation::{ - map::{MapParseTo, MapToString}, + map::{MapParseTo, MapToString, StripExt}, resolutions::average::AverageLowerResolution, }, local_db::{ @@ -23,9 +24,9 @@ use crate::{ ChartProperties, Named, }; -use chrono::NaiveDate; +use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{prelude::*, DbBackend, Statement}; +use sea_orm::{DbBackend, Statement}; use super::new_txns::{NewTxnsInt, NewTxnsMonthlyInt}; @@ -33,7 +34,7 @@ pub struct TxnsSuccessRateStatement; impl StatementFromRange for TxnsSuccessRateStatement { fn get_statement( - range: Option>, + range: Option>>, completed_migrations: &BlockscoutMigrations, ) -> Statement { if completed_migrations.denormalization { @@ -81,8 +82,9 @@ impl StatementFromRange for TxnsSuccessRateStatement { } } -pub type TxnsSuccessRateRemote = - RemoteDatabaseSource>; +pub type TxnsSuccessRateRemote = RemoteDatabaseSource< + PullAllWithAndSort, +>; pub type TxnsSuccessRateRemoteString = MapToString; @@ -113,19 +115,21 @@ define_and_impl_resolution_properties!( pub type TxnsSuccessRate = DirectVecLocalDbChartSource; +type TxnsSuccessRateS = StripExt; pub type TxnsSuccessRateWeekly = DirectVecLocalDbChartSource< - MapToString, NewTxnsInt, Week>>, + MapToString, NewTxnsInt, Week>>, Batch30Weeks, WeeklyProperties, >; pub type TxnsSuccessRateMonthly = DirectVecLocalDbChartSource< - MapToString, NewTxnsInt, Month>>, + MapToString, NewTxnsInt, Month>>, Batch36Months, MonthlyProperties, >; +type TxnsSuccessRateMonthlyS = StripExt; pub type TxnsSuccessRateYearly = DirectVecLocalDbChartSource< MapToString< - AverageLowerResolution, NewTxnsMonthlyInt, Year>, + AverageLowerResolution, NewTxnsMonthlyInt, Year>, >, Batch30Years, YearlyProperties, diff --git a/stats/stats/src/charts/lines/verified_contracts_growth.rs b/stats/stats/src/charts/lines/verified_contracts_growth.rs index 44b2f2dc1..6a2100771 100644 --- a/stats/stats/src/charts/lines/verified_contracts_growth.rs +++ b/stats/stats/src/charts/lines/verified_contracts_growth.rs @@ -1,7 +1,7 @@ use crate::{ charts::chart::ChartProperties, data_source::kinds::{ - data_manipulation::resolutions::last_value::LastValueLowerResolution, + data_manipulation::{map::StripExt, resolutions::last_value::LastValueLowerResolution}, local_db::{ parameters::update::batching::parameters::{Batch30Weeks, Batch30Years, Batch36Months}, DailyCumulativeLocalDbChartSource, DirectVecLocalDbChartSource, @@ -46,18 +46,20 @@ define_and_impl_resolution_properties!( pub type VerifiedContractsGrowth = DailyCumulativeLocalDbChartSource; +type VerifiedContractsGrowthS = StripExt; pub type VerifiedContractsGrowthWeekly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch30Weeks, WeeklyProperties, >; pub type VerifiedContractsGrowthMonthly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch36Months, MonthlyProperties, >; +type VerifiedContractsGrowthMonthlyS = StripExt; pub type VerifiedContractsGrowthYearly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch30Years, YearlyProperties, >; diff --git a/stats/stats/src/charts/mod.rs b/stats/stats/src/charts/mod.rs index 9b34d6c93..75e9b3c23 100644 --- a/stats/stats/src/charts/mod.rs +++ b/stats/stats/src/charts/mod.rs @@ -2,8 +2,9 @@ mod chart; pub mod counters; pub mod db_interaction; pub mod lines; +pub mod query_dispatch; pub mod types; pub use chart::{ - chart_properties_portrait, ChartKey, ChartProperties, ChartPropertiesObject, MissingDatePolicy, - Named, ResolutionKind, UpdateError, + chart_properties_portrait, ChartError, ChartKey, ChartObject, ChartProperties, + ChartPropertiesObject, MissingDatePolicy, Named, ResolutionKind, }; diff --git a/stats/stats/src/charts/query_dispatch.rs b/stats/stats/src/charts/query_dispatch.rs new file mode 100644 index 000000000..9cf11d312 --- /dev/null +++ b/stats/stats/src/charts/query_dispatch.rs @@ -0,0 +1,192 @@ +use std::{fmt::Debug, future::Future, pin::Pin, sync::Arc}; + +use chrono::{DateTime, NaiveDate, Utc}; +use entity::sea_orm_active_enums::ChartType; +use stats_proto::blockscout::stats::v1::Point; + +use crate::{ + data_source::{ + kinds::local_db::{ + parameter_traits::{CreateBehaviour, QueryBehaviour, UpdateBehaviour}, + LocalDbChartSource, + }, + DataSource, UpdateContext, + }, + range::{exclusive_range_to_inclusive, UniversalRange}, + RequestedPointsLimit, +}; + +use super::{ + types::{ExtendedTimespanValue, Timespan, TimespanValue}, + ChartError, ChartProperties, +}; + +/// Data query trait with unified data format (for external use) +pub trait QuerySerialized { + /// Currently `Point` or `Vec` + type Output: Send; + + /// `new` function that is created solely for the purposes of + /// dynamic dispatch (see where it's used). + fn new_for_dynamic_dispatch() -> Self + where + Self: Sized; + + /// Retrieve chart data from local storage. + fn query_data<'a>( + &self, + cx: &UpdateContext<'a>, + range: UniversalRange>, + points_limit: Option, + fill_missing_dates: bool, + ) -> Pin> + Send + 'a>>; + + /// Retrieve chart data from local storage. + fn query_data_static<'a>( + cx: &UpdateContext<'a>, + range: UniversalRange>, + points_limit: Option, + fill_missing_dates: bool, + ) -> Pin> + Send + 'a>> + where + Self: Sized, + { + Self::new_for_dynamic_dispatch().query_data(cx, range, points_limit, fill_missing_dates) + } +} + +/// [`QuerySerialized`] but for dynamic dispatch +pub type QuerySerializedDyn = Arc + Send + Sync>>; + +pub type CounterHandle = QuerySerializedDyn>; +pub type LineHandle = QuerySerializedDyn>; + +#[derive(Clone)] +pub enum ChartTypeSpecifics { + Counter { query: CounterHandle }, + Line { query: LineHandle }, +} + +impl Debug for ChartTypeSpecifics { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Counter { query: _ } => write!(f, "Counter"), + Self::Line { query: _ } => write!(f, "Line"), + } + } +} + +impl ChartTypeSpecifics { + pub fn as_chart_type(&self) -> ChartType { + match self { + Self::Counter { query: _ } => ChartType::Counter, + Self::Line { query: _ } => ChartType::Line, + } + } + + pub fn into_counter_handle(self) -> Option { + match self { + Self::Counter { query } => Some(query), + _ => None, + } + } + + pub fn into_line_handle(self) -> Option { + match self { + Self::Line { query } => Some(query), + _ => None, + } + } +} + +impl From for ChartTypeSpecifics { + fn from(val: CounterHandle) -> Self { + ChartTypeSpecifics::Counter { query: val } + } +} + +impl From for ChartTypeSpecifics { + fn from(val: LineHandle) -> Self { + ChartTypeSpecifics::Line { query: val } + } +} + +pub trait SerializableQueryOutput { + type Serialized; + fn serialize(self) -> Self::Serialized; +} + +impl SerializableQueryOutput + for Vec> +{ + type Serialized = Vec; + + fn serialize(self) -> Self::Serialized { + serialize_line_points(self) + } +} + +impl SerializableQueryOutput for TimespanValue { + type Serialized = TimespanValue; + + fn serialize(self) -> Self::Serialized { + self + } +} + +impl QuerySerialized + for LocalDbChartSource +where + MainDep: DataSource + Sync, + ResolutionDep: DataSource + Sync, + Create: CreateBehaviour + Sync, + Update: UpdateBehaviour + Sync, + Query: QueryBehaviour + Sync, + QueryOutput: SerializableQueryOutput, + QueryOutput::Serialized: Send, + ChartProps: ChartProperties, + ChartProps::Resolution: Ord + Clone + Debug, +{ + type Output = QueryOutput::Serialized; + + fn query_data<'a>( + &self, + cx: &UpdateContext<'a>, + range: UniversalRange>, + points_limit: Option, + fill_missing_dates: bool, + ) -> Pin> + Send + 'a>> + { + let cx = cx.clone(); + Box::pin(async move { + let data = Query::query_data(&cx, range, points_limit, fill_missing_dates).await?; + Ok(data.serialize()) + }) + } + + fn new_for_dynamic_dispatch() -> Self + where + Self: Sized, + { + Self(std::marker::PhantomData) + } +} + +fn serialize_point( + point: ExtendedTimespanValue, +) -> Point { + let time_range = exclusive_range_to_inclusive(point.timespan.into_time_range()); + let date_range = { time_range.start().date_naive()..=time_range.end().date_naive() }; + Point { + date: date_range.start().to_string(), + date_to: date_range.end().to_string(), + value: point.value, + is_approximate: point.is_approximate, + } +} + +pub fn serialize_line_points( + data: Vec>, +) -> Vec { + data.into_iter().map(serialize_point).collect() +} diff --git a/stats/stats/src/charts/types/timespans/day.rs b/stats/stats/src/charts/types/timespans/day.rs index b3544a9a7..4c02f4408 100644 --- a/stats/stats/src/charts/types/timespans/day.rs +++ b/stats/stats/src/charts/types/timespans/day.rs @@ -37,20 +37,26 @@ impl Timespan for NaiveDate { day_start(self) } - fn saturating_add(&self, duration: TimespanDuration) -> Self + fn checked_add(&self, duration: TimespanDuration) -> Option where Self: Sized, { self.checked_add_days(Days::new(duration.repeats())) - .unwrap_or(NaiveDate::MAX) } - fn saturating_sub(&self, duration: TimespanDuration) -> Self + fn checked_sub(&self, duration: TimespanDuration) -> Option where Self: Sized, { self.checked_sub_days(Days::new(duration.repeats())) - .unwrap_or(NaiveDate::MIN) + } + + fn max() -> Self { + NaiveDate::MAX + } + + fn min() -> Self { + NaiveDate::MIN } } diff --git a/stats/stats/src/charts/types/timespans/month.rs b/stats/stats/src/charts/types/timespans/month.rs index df1e2bf9c..baabf9dc4 100644 --- a/stats/stats/src/charts/types/timespans/month.rs +++ b/stats/stats/src/charts/types/timespans/month.rs @@ -73,30 +73,32 @@ impl Timespan for Month { self.saturating_first_day().saturating_start_timestamp() } - fn saturating_add(&self, duration: TimespanDuration) -> Self + fn checked_add(&self, duration: TimespanDuration) -> Option where Self: Sized, { - let result_month_date = self - .date_in_month - .checked_add_months(chrono::Months::new( - duration.repeats().try_into().unwrap_or(u32::MAX), - )) - .unwrap_or(NaiveDate::MAX); - Self::from_date(result_month_date) + let duration = chrono::Months::new(duration.repeats().try_into().ok()?); + self.date_in_month + .checked_add_months(duration) + .map(Self::from_date) } - fn saturating_sub(&self, duration: TimespanDuration) -> Self + fn checked_sub(&self, duration: TimespanDuration) -> Option where Self: Sized, { - let result_month_date = self - .date_in_month - .checked_sub_months(chrono::Months::new( - duration.repeats().try_into().unwrap_or(u32::MAX), - )) - .unwrap_or(NaiveDate::MIN); - Self::from_date(result_month_date) + let duration = chrono::Months::new(duration.repeats().try_into().ok()?); + self.date_in_month + .checked_sub_months(duration) + .map(Self::from_date) + } + + fn max() -> Self { + Self::from_date(NaiveDate::MAX) + } + + fn min() -> Self { + Self::from_date(NaiveDate::MIN) } } @@ -218,7 +220,7 @@ mod tests { } #[test] - fn month_saturating_arithmetics_works() { + fn month_arithmetics_works() { assert_eq!( Month::from_date(d("2015-06-01")) .saturating_add(TimespanDuration::from_timespan_repeats(3)), @@ -229,6 +231,16 @@ mod tests { .saturating_sub(TimespanDuration::from_timespan_repeats(3)), Month::from_date(d("2015-03-01")) ); + assert_eq!( + Month::from_date(d("2015-06-01")) + .checked_add(TimespanDuration::from_timespan_repeats(3)), + Some(Month::from_date(d("2015-09-01"))) + ); + assert_eq!( + Month::from_date(d("2015-06-01")) + .checked_sub(TimespanDuration::from_timespan_repeats(3)), + Some(Month::from_date(d("2015-03-01"))) + ); assert_eq!( Month::from_date(d("2015-06-01")) @@ -241,6 +253,17 @@ mod tests { Month::from_date(NaiveDate::MIN) ); + assert_eq!( + Month::from_date(d("2015-06-01")) + .checked_add(TimespanDuration::from_timespan_repeats(u64::MAX)), + None + ); + assert_eq!( + Month::from_date(d("2015-06-01")) + .checked_sub(TimespanDuration::from_timespan_repeats(u64::MAX)), + None + ); + assert_eq!( Month::from_date(NaiveDate::MAX) .saturating_add(TimespanDuration::from_timespan_repeats(1)), @@ -251,6 +274,17 @@ mod tests { .saturating_sub(TimespanDuration::from_timespan_repeats(1)), Month::from_date(NaiveDate::MIN) ); + + assert_eq!( + Month::from_date(NaiveDate::MAX) + .checked_add(TimespanDuration::from_timespan_repeats(1)), + None + ); + assert_eq!( + Month::from_date(NaiveDate::MIN) + .checked_sub(TimespanDuration::from_timespan_repeats(1)), + None + ); } #[test] diff --git a/stats/stats/src/charts/types/timespans/week.rs b/stats/stats/src/charts/types/timespans/week.rs index 3ddf83c56..3ac83c2e9 100644 --- a/stats/stats/src/charts/types/timespans/week.rs +++ b/stats/stats/src/charts/types/timespans/week.rs @@ -88,26 +88,32 @@ impl Timespan for Week { self.saturating_first_day().saturating_start_timestamp() } - fn saturating_add(&self, duration: TimespanDuration) -> Self + fn checked_add(&self, duration: TimespanDuration) -> Option where Self: Sized, { - let result_week_date = self - .saturating_first_day() - .checked_add_days(Days::new(duration.repeats() * 7)) - .unwrap_or(NaiveDate::MAX); - Self::from_date(result_week_date) + let duration = Days::new(duration.repeats().checked_mul(7)?); + self.saturating_first_day() + .checked_add_days(duration) + .map(Self::from_date) } - fn saturating_sub(&self, duration: TimespanDuration) -> Self + fn checked_sub(&self, duration: TimespanDuration) -> Option where Self: Sized, { - let result_week_date = self - .saturating_first_day() - .checked_sub_days(Days::new(duration.repeats() * 7)) - .unwrap_or(NaiveDate::MIN); - Self::from_date(result_week_date) + let duration = Days::new(duration.repeats().checked_mul(7)?); + self.saturating_first_day() + .checked_sub_days(duration) + .map(Self::from_date) + } + + fn max() -> Self { + Self::from_date(NaiveDate::MAX) + } + + fn min() -> Self { + Self::from_date(NaiveDate::MIN) } } diff --git a/stats/stats/src/charts/types/timespans/year.rs b/stats/stats/src/charts/types/timespans/year.rs index 793cf4b49..67f37f92f 100644 --- a/stats/stats/src/charts/types/timespans/year.rs +++ b/stats/stats/src/charts/types/timespans/year.rs @@ -55,13 +55,17 @@ impl Year { self.clamp_by_naive_date_range().0 } + fn year_number_within_naive_date(year: i32) -> bool { + NaiveDate::from_yo_opt(year, 1).is_some() + } + fn clamp_by_naive_date_range(self) -> Self { - if NaiveDate::from_yo_opt(self.0, 1).is_some() { + if Self::year_number_within_naive_date(self.0) { self } else if self.0 > 0 { - Self::from_date(NaiveDate::MAX) + ::max() } else { - Self::from_date(NaiveDate::MIN) + ::min() } } } @@ -98,6 +102,40 @@ impl Timespan for Year { let sub_years: i32 = duration.repeats().try_into().unwrap_or(i32::MAX); Self(self.number_within_naive_date().saturating_sub(sub_years)).clamp_by_naive_date_range() } + + fn checked_add(&self, duration: TimespanDuration) -> Option + where + Self: Sized, + { + let add_years = duration.repeats().try_into().ok()?; + let new_year_num = self.number_within_naive_date().checked_add(add_years)?; + if Self::year_number_within_naive_date(new_year_num) { + Some(Self(new_year_num)) + } else { + None + } + } + + fn checked_sub(&self, duration: TimespanDuration) -> Option + where + Self: Sized, + { + let sub_years: i32 = duration.repeats().try_into().ok()?; + let new_year_num = self.number_within_naive_date().checked_sub(sub_years)?; + if Self::year_number_within_naive_date(new_year_num) { + Some(Self(new_year_num)) + } else { + None + } + } + + fn max() -> Self { + Self::from_date(NaiveDate::MAX) + } + + fn min() -> Self { + Self::from_date(NaiveDate::MIN) + } } impl ConsistsOf for Year { @@ -254,6 +292,53 @@ mod tests { Year::from_date(NaiveDate::MIN) ); } + #[test] + fn year_checked_add_works() { + assert_eq!( + Year(2016).checked_add(TimespanDuration::from_timespan_repeats(16)), + Some(Year(2032)) + ); + assert_eq!( + Year(2016).checked_add(TimespanDuration::from_timespan_repeats(0)), + Some(Year(2016)) + ); + assert_eq!( + Year(2016).checked_add(TimespanDuration::from_timespan_repeats(u64::MAX)), + None + ); + assert_eq!( + Year::from_date(NaiveDate::MAX).checked_add(TimespanDuration::from_timespan_repeats(1)), + None + ); + assert_eq!( + Year(i32::MAX - 5).checked_add(TimespanDuration::from_timespan_repeats(3)), + None + ); + } + + #[test] + fn year_checked_sub_works() { + assert_eq!( + Year(2016).checked_sub(TimespanDuration::from_timespan_repeats(16)), + Some(Year(2000)) + ); + assert_eq!( + Year(2016).checked_sub(TimespanDuration::from_timespan_repeats(0)), + Some(Year(2016)) + ); + assert_eq!( + Year(2016).checked_sub(TimespanDuration::from_timespan_repeats(u64::MAX)), + None + ); + assert_eq!( + Year::from_date(NaiveDate::MIN).checked_sub(TimespanDuration::from_timespan_repeats(1)), + None + ); + assert_eq!( + Year(i32::MIN + 5).checked_sub(TimespanDuration::from_timespan_repeats(3)), + None + ); + } #[test] fn year_saturating_first_timestamp_works() { diff --git a/stats/stats/src/charts/types/traits.rs b/stats/stats/src/charts/types/traits.rs index 1afe9b83a..82a795982 100644 --- a/stats/stats/src/charts/types/traits.rs +++ b/stats/stats/src/charts/types/traits.rs @@ -44,12 +44,26 @@ pub trait Timespan { self.saturating_start_timestamp() ..self.saturating_next_timespan().saturating_start_timestamp() } - fn saturating_add(&self, duration: TimespanDuration) -> Self + fn checked_add(&self, duration: TimespanDuration) -> Option where Self: Sized; - fn saturating_sub(&self, duration: TimespanDuration) -> Self + fn checked_sub(&self, duration: TimespanDuration) -> Option where Self: Sized; + fn max() -> Self; + fn min() -> Self; + fn saturating_add(&self, duration: TimespanDuration) -> Self + where + Self: Sized, + { + self.checked_add(duration).unwrap_or_else(|| Self::max()) + } + fn saturating_sub(&self, duration: TimespanDuration) -> Self + where + Self: Sized, + { + self.checked_sub(duration).unwrap_or_else(|| Self::min()) + } } // if for some rare reason trait is needed diff --git a/stats/stats/src/data_processing.rs b/stats/stats/src/data_processing.rs index 99195854a..73cc1e3d1 100644 --- a/stats/stats/src/data_processing.rs +++ b/stats/stats/src/data_processing.rs @@ -3,7 +3,7 @@ use chrono::NaiveDate; use crate::{ charts::types::timespans::DateValue, types::{Timespan, TimespanValue}, - UpdateError, + ChartError, }; use std::{ mem, @@ -17,7 +17,7 @@ use std::{ pub fn cumsum( mut data: Vec>, mut prev_sum: Value, -) -> Result>, UpdateError> +) -> Result>, ChartError> where Value: AddAssign + Clone, TimespanValue: Default, @@ -40,7 +40,7 @@ where pub fn deltas( mut data: Vec>, mut prev_value: Value, -) -> Result>, UpdateError> +) -> Result>, ChartError> where Value: SubAssign + Clone, TimespanValue: Default, @@ -61,7 +61,7 @@ where pub fn sum( data: &[TimespanValue], mut partial_sum: Value, -) -> Result, UpdateError> +) -> Result, ChartError> where Resolution: Timespan + Clone + Ord, Value: AddAssign + Clone, diff --git a/stats/stats/src/data_source/kinds/auxiliary/cumulative.rs b/stats/stats/src/data_source/kinds/auxiliary/cumulative.rs index dc21d253d..ca77c9c99 100644 --- a/stats/stats/src/data_source/kinds/auxiliary/cumulative.rs +++ b/stats/stats/src/data_source/kinds/auxiliary/cumulative.rs @@ -1,7 +1,4 @@ -use std::{ - marker::PhantomData, - ops::{AddAssign, Range}, -}; +use std::{marker::PhantomData, ops::AddAssign}; use blockscout_metrics_tools::AggregateTimer; use chrono::{DateTime, Utc}; @@ -11,8 +8,9 @@ use sea_orm::DatabaseConnection; use crate::{ data_processing::cumsum, data_source::{DataSource, UpdateContext}, + range::UniversalRange, types::TimespanValue, - UpdateError, + ChartError, }; /// Auxiliary source for cumulative chart. @@ -45,16 +43,16 @@ where Ok(()) } - async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), UpdateError> { + async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), ChartError> { // just an adapter; inner is handled recursively Ok(()) } async fn query_data( cx: &UpdateContext<'_>, - range: Option>>, + range: UniversalRange>, dependency_data_fetch_timer: &mut AggregateTimer, - ) -> Result { + ) -> Result { let delta_data = Delta::query_data(cx, range, dependency_data_fetch_timer).await?; let data = cumsum::(delta_data, Value::zero())?; Ok(data) diff --git a/stats/stats/src/data_source/kinds/data_manipulation/delta.rs b/stats/stats/src/data_source/kinds/data_manipulation/delta.rs index 9b6edb65d..9aee44706 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/delta.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/delta.rs @@ -3,23 +3,19 @@ //! //! I.e. chart "New accounts" is a delta of "Total accounts". -use std::{ - fmt::Display, - marker::PhantomData, - ops::{Range, SubAssign}, - str::FromStr, -}; +use std::{fmt::Display, marker::PhantomData, ops::SubAssign, str::FromStr}; use blockscout_metrics_tools::AggregateTimer; use chrono::{DateTime, TimeDelta, Utc}; use rust_decimal::prelude::Zero; -use sea_orm::{prelude::DateTimeUtc, DatabaseConnection, DbErr}; +use sea_orm::{DatabaseConnection, DbErr}; use crate::{ data_processing::deltas, data_source::{DataSource, UpdateContext}, + range::UniversalRange, types::TimespanValue, - UpdateError, + ChartError, }; /// Calculate delta data from cumulative dependency. @@ -56,26 +52,25 @@ where Ok(()) } - async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), UpdateError> { + async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), ChartError> { // just an adapter; inner is handled recursively Ok(()) } async fn query_data( cx: &UpdateContext<'_>, - range: Option>, + range: UniversalRange>, dependency_data_fetch_timer: &mut AggregateTimer, - ) -> Result { - let request_range = range.clone().map(|r| { - let start = r - .start - .checked_sub_signed(TimeDelta::days(1)) - .unwrap_or(DateTime::::MAX_UTC); - let end = r.end; - start..end + ) -> Result { + let mut request_range = range.clone(); + request_range.start = request_range.start.map(|s| { + s.checked_sub_signed(TimeDelta::days(1)) + .unwrap_or(DateTime::::MAX_UTC) }); + let start_is_bounded = request_range.start.is_some(); + let cum_data = DS::query_data(cx, request_range, dependency_data_fetch_timer).await?; - let (prev_value, cum_data) = if range.is_some() { + let (prev_value, cum_data) = if start_is_bounded { let mut cum_data = cum_data.into_iter(); let Some(range_start) = cum_data.next() else { tracing::warn!("Value before the range was not found, finishing update"); diff --git a/stats/stats/src/data_source/kinds/data_manipulation/filter_deducible.rs b/stats/stats/src/data_source/kinds/data_manipulation/filter_deducible.rs index 7b4b5786d..a7b4581b7 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/filter_deducible.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/filter_deducible.rs @@ -1,7 +1,7 @@ //! Filter points that can be deduced according to `MissingDatePolicy`. //! Can help with space usage efficiency. -use std::{marker::PhantomData, ops::Range}; +use std::marker::PhantomData; use blockscout_metrics_tools::AggregateTimer; use chrono::{DateTime, Utc}; @@ -10,8 +10,9 @@ use sea_orm::DatabaseConnection; use crate::{ data_source::{DataSource, UpdateContext}, + range::UniversalRange, types::TimespanValue, - ChartProperties, MissingDatePolicy, UpdateError, + ChartError, ChartProperties, MissingDatePolicy, }; /// Pass only essential points from `D`, removing ones that can be deduced @@ -44,16 +45,16 @@ where Ok(()) } - async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), UpdateError> { + async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), ChartError> { // just an adapter; inner is handled recursively Ok(()) } async fn query_data( cx: &UpdateContext<'_>, - range: Option>>, + range: UniversalRange>, dependency_data_fetch_timer: &mut AggregateTimer, - ) -> Result { + ) -> Result { let data = DS::query_data(cx, range, dependency_data_fetch_timer).await?; Ok(match Properties::missing_date_policy() { MissingDatePolicy::FillZero => { @@ -79,12 +80,16 @@ where #[cfg(test)] mod tests { + use std::sync::Arc; + use crate::{ data_source::types::BlockscoutMigrations, gettable_const, lines::PredefinedMockSource, + range::UniversalRange, tests::point_construction::{d_v_double, dt}, types::timespans::DateValue, + utils::MarkedDbConnection, MissingDatePolicy, Named, }; @@ -151,7 +156,9 @@ mod tests { type TestedPrevious = FilterDeducible; // db is not used in mock - let empty_db = sea_orm::Database::connect("sqlite::memory:").await.unwrap(); + let empty_db = MarkedDbConnection::in_memory(Arc::new( + sea_orm::Database::connect("sqlite::memory:").await.unwrap(), + )); let context = UpdateContext { db: &empty_db, @@ -161,9 +168,13 @@ mod tests { force_full: false, }; assert_eq!( - ::query_data(&context, None, &mut AggregateTimer::new()) - .await - .unwrap(), + ::query_data( + &context, + UniversalRange::full(), + &mut AggregateTimer::new() + ) + .await + .unwrap(), vec![ d_v_double("2024-07-08", 5.0), d_v_double("2024-07-10", 5.0), @@ -175,9 +186,13 @@ mod tests { ] ); assert_eq!( - ::query_data(&context, None, &mut AggregateTimer::new()) - .await - .unwrap(), + ::query_data( + &context, + UniversalRange::full(), + &mut AggregateTimer::new() + ) + .await + .unwrap(), vec![ d_v_double("2024-07-08", 5.0), d_v_double("2024-07-14", 10.3), diff --git a/stats/stats/src/data_source/kinds/data_manipulation/last_point.rs b/stats/stats/src/data_source/kinds/data_manipulation/last_point.rs index ff98bfcf1..c3fcb11ad 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/last_point.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/last_point.rs @@ -2,18 +2,18 @@ //! //! Takes last data point from some other (vector) source -use std::{marker::PhantomData, ops::Range}; +use std::marker::PhantomData; use blockscout_metrics_tools::AggregateTimer; use chrono::{DateTime, Utc}; -use sea_orm::{prelude::DateTimeUtc, DatabaseConnection, DbErr}; +use sea_orm::{DatabaseConnection, DbErr}; use crate::{ - charts::ChartProperties, data_source::{source::DataSource, UpdateContext}, + range::UniversalRange, types::{Timespan, TimespanValue, ZeroTimespanValue}, utils::day_start, - UpdateError, + ChartError, }; pub struct LastPoint(PhantomData) @@ -24,7 +24,7 @@ impl DataSource for LastPoint where Resolution: Timespan + Ord + Send, Value: Send, - DS: DataSource>> + ChartProperties, + DS: DataSource>>, TimespanValue: ZeroTimespanValue, { type MainDependencies = DS; @@ -42,19 +42,19 @@ where Ok(()) } - async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), UpdateError> { + async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), ChartError> { // just an adapter; inner is handled recursively Ok(()) } async fn query_data( cx: &UpdateContext<'_>, - _range: Option>, + _range: UniversalRange>, dependency_data_fetch_timer: &mut AggregateTimer, - ) -> Result { + ) -> Result { let data = DS::query_data( cx, - Some(day_start(&cx.time.date_naive())..cx.time), + (day_start(&cx.time.date_naive())..cx.time).into(), dependency_data_fetch_timer, ) .await?; diff --git a/stats/stats/src/data_source/kinds/data_manipulation/map/mod.rs b/stats/stats/src/data_source/kinds/data_manipulation/map/mod.rs index e48fffc0f..a0d114b64 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/map/mod.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/map/mod.rs @@ -1,21 +1,24 @@ //! `map` for a data source, i.e. applies a function to the output //! of some other source. -use std::{marker::PhantomData, ops::Range}; +use std::marker::PhantomData; use blockscout_metrics_tools::AggregateTimer; -use chrono::Utc; -use sea_orm::{prelude::DateTimeUtc, DatabaseConnection, DbErr}; +use chrono::{DateTime, Utc}; +use sea_orm::{DatabaseConnection, DbErr}; use crate::{ data_source::{DataSource, UpdateContext}, - UpdateError, + range::UniversalRange, + ChartError, }; mod parse; +mod strip_extension; mod to_string; pub use parse::MapParseTo; +pub use strip_extension::StripExt; pub use to_string::MapToString; /// Apply `F` to each value queried from data source `D` @@ -26,7 +29,7 @@ where pub trait MapFunction { type Output: Send; - fn function(inner_data: Input) -> Result; + fn function(inner_data: Input) -> Result; } impl DataSource for Map @@ -49,16 +52,16 @@ where Ok(()) } - async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), UpdateError> { + async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), ChartError> { // just an adapter; inner is handled recursively Ok(()) } async fn query_data( cx: &UpdateContext<'_>, - range: Option>, + range: UniversalRange>, dependency_data_fetch_timer: &mut AggregateTimer, - ) -> Result { + ) -> Result { let inner_data = ::query_data(cx, range, dependency_data_fetch_timer).await?; F::function(inner_data) diff --git a/stats/stats/src/data_source/kinds/data_manipulation/map/parse.rs b/stats/stats/src/data_source/kinds/data_manipulation/map/parse.rs index bac618b32..dfe3f5fd4 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/map/parse.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/map/parse.rs @@ -1,7 +1,7 @@ use std::{fmt::Display, marker::PhantomData, str::FromStr}; use crate::{ - data_source::kinds::data_manipulation::map::MapFunction, types::TimespanValue, UpdateError, + data_source::kinds::data_manipulation::map::MapFunction, types::TimespanValue, ChartError, }; use super::Map; @@ -19,19 +19,19 @@ where fn function( inner_data: Vec>, - ) -> Result>, UpdateError> { + ) -> Result>, ChartError> { inner_data .into_iter() .map(|p| { let val_parsed = p.value.parse::().map_err(|e| { - UpdateError::Internal(format!("failed to parse values of dependency: {e}")) + ChartError::Internal(format!("failed to parse values of dependency: {e}")) })?; Ok(TimespanValue { timespan: p.timespan, value: val_parsed, }) }) - .collect::, UpdateError>>() + .collect::, ChartError>>() } } @@ -43,11 +43,9 @@ where { type Output = TimespanValue; - fn function( - inner_data: TimespanValue, - ) -> Result { + fn function(inner_data: TimespanValue) -> Result { let val_parsed = inner_data.value.parse::().map_err(|e| { - UpdateError::Internal(format!("failed to parse values of dependency: {e}")) + ChartError::Internal(format!("failed to parse values of dependency: {e}")) })?; Ok(TimespanValue { timespan: inner_data.timespan, diff --git a/stats/stats/src/data_source/kinds/data_manipulation/map/strip_extension.rs b/stats/stats/src/data_source/kinds/data_manipulation/map/strip_extension.rs new file mode 100644 index 000000000..f0accab13 --- /dev/null +++ b/stats/stats/src/data_source/kinds/data_manipulation/map/strip_extension.rs @@ -0,0 +1,36 @@ +use crate::{ + types::{ExtendedTimespanValue, TimespanValue}, + ChartError, +}; + +use super::{Map, MapFunction}; + +/// Remove `Extended` part from `ExtendedTimespanValue`. +/// Used because it's easier to impl and use other data source modifiers +/// this way. +pub struct StripExtensionFunction; + +impl MapFunction>> for StripExtensionFunction +where + R: Send, + V: Send, +{ + type Output = Vec>; + fn function(inner_data: Vec>) -> Result { + Ok(inner_data.into_iter().map(|p| p.into()).collect()) + } +} + +impl MapFunction> for StripExtensionFunction +where + R: Send, + V: Send, +{ + type Output = TimespanValue; + fn function(inner_data: ExtendedTimespanValue) -> Result { + Ok(inner_data.into()) + } +} + +/// Remove `Extended` part from `ExtendedTimespanValue`(-s) +pub type StripExt = Map; diff --git a/stats/stats/src/data_source/kinds/data_manipulation/map/to_string.rs b/stats/stats/src/data_source/kinds/data_manipulation/map/to_string.rs index ccf6a3d11..bf6b2dc98 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/map/to_string.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/map/to_string.rs @@ -1,4 +1,4 @@ -use crate::{types::TimespanValue, UpdateError}; +use crate::{types::TimespanValue, ChartError}; use super::{Map, MapFunction}; @@ -12,7 +12,7 @@ where type Output = Vec>; fn function( inner_data: Vec>, - ) -> Result { + ) -> Result { Ok(inner_data.into_iter().map(|p| p.into()).collect()) } } @@ -23,7 +23,7 @@ where TimespanValue: Into>, { type Output = TimespanValue; - fn function(inner_data: TimespanValue) -> Result { + fn function(inner_data: TimespanValue) -> Result { Ok(inner_data.into()) } } diff --git a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/average.rs b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/average.rs index 7bc8dc642..1b96272c6 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/average.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/average.rs @@ -1,17 +1,18 @@ //! Constructors for lower resolutions of average value charts -use std::{cmp::Ordering, fmt::Debug, marker::PhantomData, ops::Range}; +use std::{cmp::Ordering, fmt::Debug, marker::PhantomData}; use blockscout_metrics_tools::AggregateTimer; use chrono::{DateTime, Utc}; use itertools::{EitherOrBoth, Itertools}; -use sea_orm::{prelude::DateTimeUtc, DatabaseConnection, DbErr}; +use sea_orm::{DatabaseConnection, DbErr}; use crate::{ data_source::{ kinds::data_manipulation::resolutions::reduce_each_timespan, DataSource, UpdateContext, }, + range::UniversalRange, types::{ConsistsOf, Timespan, TimespanValue}, - UpdateError, + ChartError, }; use super::extend_to_timespan_boundaries; @@ -34,7 +35,7 @@ impl DataSource where Average: DataSource>>, Weight: DataSource>>, - LowerRes: Timespan + ConsistsOf + Eq + Debug + Send, + LowerRes: Timespan + ConsistsOf + Eq + Ord + Debug + Send, HigherRes: Ord + Clone + Debug + Send, { type MainDependencies = Average; @@ -54,17 +55,17 @@ where Ok(()) } - async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), UpdateError> { + async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), ChartError> { // just an adapter; inner is handled recursively Ok(()) } async fn query_data( cx: &UpdateContext<'_>, - range: Option>, + range: UniversalRange>, dependency_data_fetch_timer: &mut AggregateTimer, - ) -> Result { - let time_range_for_lower_res = range.map(extend_to_timespan_boundaries::); + ) -> Result { + let time_range_for_lower_res = extend_to_timespan_boundaries::(range); let high_res_averages = Average::query_data( cx, time_range_for_lower_res.clone(), @@ -188,12 +189,15 @@ where #[cfg(test)] mod tests { + use std::{ops::Range, sync::Arc}; + use crate::{ data_source::{kinds::data_manipulation::map::MapParseTo, types::BlockscoutMigrations}, gettable_const, lines::{PredefinedMockSource, PseudoRandomMockRetrieve}, tests::point_construction::{d, d_v, d_v_double, d_v_int, dt, w_v_double, week_of}, types::timespans::{DateValue, Week, WeekValue}, + utils::MarkedDbConnection, MissingDatePolicy, }; @@ -316,7 +320,9 @@ mod tests { // 8-14, 15-21, 22-28 // db is not used in mock - let db = sea_orm::Database::connect("sqlite::memory:").await.unwrap(); + let db = MarkedDbConnection::in_memory(Arc::new( + sea_orm::Database::connect("sqlite::memory:").await.unwrap(), + )); let output: Vec> = TestedAverageSource::query_data( &UpdateContext { db: &db, @@ -325,7 +331,7 @@ mod tests { time: dt("2024-07-15T09:00:00").and_utc(), force_full: false, }, - Some(dt("2024-07-08T09:00:00").and_utc()..dt("2024-07-15T00:00:01").and_utc()), + (dt("2024-07-08T09:00:00").and_utc()..dt("2024-07-15T00:00:01").and_utc()).into(), &mut AggregateTimer::new(), ) .await @@ -364,7 +370,9 @@ mod tests { AverageLowerResolution; // db is not used in mock - let empty_db = sea_orm::Database::connect("sqlite::memory:").await.unwrap(); + let empty_db = MarkedDbConnection::in_memory(Arc::new( + sea_orm::Database::connect("sqlite::memory:").await.unwrap(), + )); let context = UpdateContext { db: &empty_db, @@ -375,9 +383,13 @@ mod tests { }; let week_1_average = (5.0 * 100.0 + 34.2 * 2.0 + 10.3 * 12.0) / (100.0 + 2.0 + 12.0); assert_eq!( - TestedAverageSource::query_data(&context, None, &mut AggregateTimer::new()) - .await - .unwrap(), + TestedAverageSource::query_data( + &context, + UniversalRange::full(), + &mut AggregateTimer::new() + ) + .await + .unwrap(), vec![ w_v_double("2024-07-08", week_1_average), w_v_double("2024-07-15", 5.0) @@ -410,7 +422,9 @@ mod tests { AverageLowerResolution; // db is not used in mock - let empty_db = sea_orm::Database::connect("sqlite::memory:").await.unwrap(); + let empty_db = MarkedDbConnection::in_memory(Arc::new( + sea_orm::Database::connect("sqlite::memory:").await.unwrap(), + )); let context = UpdateContext { db: &empty_db, @@ -420,9 +434,13 @@ mod tests { force_full: false, }; assert_eq!( - TestedAverageSource::query_data(&context, None, &mut AggregateTimer::new()) - .await - .unwrap(), + TestedAverageSource::query_data( + &context, + UniversalRange::full(), + &mut AggregateTimer::new() + ) + .await + .unwrap(), vec![w_v_double("2022-11-07", 0.8888888888888888),] ); } @@ -452,7 +470,9 @@ mod tests { AverageLowerResolution; // db is not used in mock - let empty_db = sea_orm::Database::connect("sqlite::memory:").await.unwrap(); + let empty_db = MarkedDbConnection::in_memory(Arc::new( + sea_orm::Database::connect("sqlite::memory:").await.unwrap(), + )); let context = UpdateContext { db: &empty_db, @@ -462,9 +482,13 @@ mod tests { force_full: false, }; assert_eq!( - TestedAverageSource::query_data(&context, None, &mut AggregateTimer::new()) - .await - .unwrap(), + TestedAverageSource::query_data( + &context, + UniversalRange::full(), + &mut AggregateTimer::new() + ) + .await + .unwrap(), vec![w_v_double("2022-11-07", 1.0),] ); } diff --git a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/last_value.rs b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/last_value.rs index 6af0b73a7..8e14e9c74 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/last_value.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/last_value.rs @@ -3,16 +3,17 @@ //! Intended for "growth" charts where cumulative number of something //! is presented. -use std::{fmt::Debug, marker::PhantomData, ops::Range}; +use std::{fmt::Debug, marker::PhantomData}; use blockscout_metrics_tools::AggregateTimer; use chrono::{DateTime, Utc}; -use sea_orm::{prelude::DateTimeUtc, DatabaseConnection, DbErr}; +use sea_orm::{DatabaseConnection, DbErr}; use crate::{ data_source::{DataSource, UpdateContext}, + range::UniversalRange, types::{ConsistsOf, Timespan, TimespanValue}, - UpdateError, + ChartError, }; use super::{extend_to_timespan_boundaries, reduce_each_timespan}; @@ -22,7 +23,7 @@ pub struct LastValueLowerResolution(PhantomData<(DS, LowerRes)>); impl DataSource for LastValueLowerResolution where - LowerRes: Timespan + ConsistsOf + Eq + Send, + LowerRes: Timespan + ConsistsOf + Eq + Ord + Send, HigherRes: Clone, Value: Send + Debug, DS: DataSource>>, @@ -44,23 +45,19 @@ where Ok(()) } - async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), UpdateError> { + async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), ChartError> { // just an adapter; inner is handled recursively Ok(()) } async fn query_data( cx: &UpdateContext<'_>, - range: Option>, + range: UniversalRange>, dependency_data_fetch_timer: &mut AggregateTimer, - ) -> Result { - let time_range_for_lower_res = range.map(extend_to_timespan_boundaries::); - let high_res_data = DS::query_data( - cx, - time_range_for_lower_res.clone(), - dependency_data_fetch_timer, - ) - .await?; + ) -> Result { + let time_range_for_lower_res = extend_to_timespan_boundaries::(range); + let high_res_data = + DS::query_data(cx, time_range_for_lower_res, dependency_data_fetch_timer).await?; Ok(reduce_each_timespan( high_res_data, |t| LowerRes::from_smaller(t.timespan.clone()), @@ -80,6 +77,8 @@ where #[cfg(test)] mod tests { + use std::sync::Arc; + use blockscout_metrics_tools::AggregateTimer; use pretty_assertions::assert_eq; @@ -87,8 +86,10 @@ mod tests { data_source::{types::BlockscoutMigrations, DataSource, UpdateContext}, gettable_const, lines::PredefinedMockSource, + range::UniversalRange, tests::point_construction::{d_v_int, dt, w_v_int}, types::timespans::{DateValue, Week}, + utils::MarkedDbConnection, MissingDatePolicy, }; @@ -111,7 +112,9 @@ mod tests { type MockSourceWeekly = LastValueLowerResolution; // db is not used in mock - let empty_db = sea_orm::Database::connect("sqlite::memory:").await.unwrap(); + let empty_db = MarkedDbConnection::in_memory(Arc::new( + sea_orm::Database::connect("sqlite::memory:").await.unwrap(), + )); let context = UpdateContext { db: &empty_db, @@ -121,7 +124,7 @@ mod tests { force_full: false, }; assert_eq!( - MockSource::query_data(&context, None, &mut AggregateTimer::new()) + MockSource::query_data(&context, UniversalRange::full(), &mut AggregateTimer::new()) .await .unwrap(), vec![ @@ -132,9 +135,13 @@ mod tests { ] ); assert_eq!( - MockSourceWeekly::query_data(&context, None, &mut AggregateTimer::new()) - .await - .unwrap(), + MockSourceWeekly::query_data( + &context, + UniversalRange::full(), + &mut AggregateTimer::new() + ) + .await + .unwrap(), vec![w_v_int("2024-07-08", 3), w_v_int("2024-07-22", 1234),] ); } diff --git a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/mod.rs b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/mod.rs index 2ceb4f462..33db3f8d6 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/mod.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/mod.rs @@ -2,34 +2,35 @@ //! type/meaning. //! E.g. "weekly average block rewards" from "daily average block rewards". -use std::ops::{Range, RangeInclusive}; - use chrono::{DateTime, Utc}; -use crate::{types::Timespan, utils::exclusive_datetime_range_to_inclusive}; +use crate::{range::UniversalRange, types::Timespan}; pub mod average; pub mod last_value; pub mod sum; // Boundaries of resulting range - timespans that contain boundaries of date range -fn date_range_to_timespan(range: Range>) -> RangeInclusive { - let range = exclusive_datetime_range_to_inclusive(range); - let start_timespan = T::from_date(range.start().date_naive()); - let end_timespan = T::from_date(range.end().date_naive()); - start_timespan..=end_timespan +fn date_range_to_timespan(range: UniversalRange>) -> UniversalRange { + let (start, end_inclusive) = range.into_inclusive_pair(); + let start_timespan = start.map(|s| T::from_date(s.date_naive())); + let end_timespan = end_inclusive.map(|e| T::from_date(e.date_naive())); + (start_timespan..=end_timespan).into() } -pub fn extend_to_timespan_boundaries( - range: Range>, -) -> Range> { +pub fn extend_to_timespan_boundaries( + range: UniversalRange>, +) -> UniversalRange> { let timespan_range = date_range_to_timespan::(range); // start of timespan containing range start - let start: DateTime = timespan_range.start().saturating_start_timestamp(); + let (start, end) = timespan_range.into_inclusive_pair(); + let start = start.map(|s| s.saturating_start_timestamp()); // start of timespan following range end (to get exclusive range again) - let timespan_after_range = timespan_range.end().saturating_next_timespan(); - let end = timespan_after_range.saturating_start_timestamp(); - start..end + let end = end.map(|e| { + let timespan_after_range = e.saturating_next_timespan(); + timespan_after_range.saturating_start_timestamp() + }); + (start..end).into() } /// Produce vector of timespan data `LResPoint` from vector of smaller timespan data `HResPoint`. @@ -87,45 +88,59 @@ mod tests { assert_eq!( date_range_to_timespan::( - dt("2024-07-08T09:00:00").and_utc()..dt("2024-07-14T09:00:00").and_utc() - ), + (dt("2024-07-08T09:00:00").and_utc()..dt("2024-07-14T09:00:00").and_utc()).into() + ) + .try_into_inclusive() + .unwrap(), week_of("2024-07-08")..=week_of("2024-07-08") ); assert_eq!( date_range_to_timespan::( - dt("2024-07-08T09:00:00").and_utc()..dt("2024-07-14T23:59:59").and_utc() - ), + (dt("2024-07-08T09:00:00").and_utc()..dt("2024-07-14T23:59:59").and_utc()).into() + ) + .try_into_inclusive() + .unwrap(), week_of("2024-07-08")..=week_of("2024-07-08") ); assert_eq!( date_range_to_timespan::( - dt("2024-07-08T09:00:00").and_utc()..dt("2024-07-15T00:00:00").and_utc() - ), + (dt("2024-07-08T09:00:00").and_utc()..dt("2024-07-15T00:00:00").and_utc()).into() + ) + .try_into_inclusive() + .unwrap(), week_of("2024-07-08")..=week_of("2024-07-08") ); assert_eq!( date_range_to_timespan::( - dt("1995-12-31T09:00:00").and_utc()..dt("1995-12-31T23:59:60").and_utc() - ), + (dt("1995-12-31T09:00:00").and_utc()..dt("1995-12-31T23:59:60").and_utc()).into() + ) + .try_into_inclusive() + .unwrap(), week_of("1995-12-31")..=week_of("1995-12-31") ); assert_eq!( date_range_to_timespan::( - dt("1995-12-31T09:00:00").and_utc()..dt("1996-01-01T00:00:00").and_utc() - ), + (dt("1995-12-31T09:00:00").and_utc()..dt("1996-01-01T00:00:00").and_utc()).into() + ) + .try_into_inclusive() + .unwrap(), week_of("1995-12-31")..=week_of("1995-12-31") ); assert_eq!( date_range_to_timespan::( - dt("2024-07-08T09:00:00").and_utc()..dt("2024-07-15T00:00:01").and_utc() - ), + (dt("2024-07-08T09:00:00").and_utc()..dt("2024-07-15T00:00:01").and_utc()).into() + ) + .try_into_inclusive() + .unwrap(), week_of("2024-07-08")..=week_of("2024-07-15") ); assert_eq!( date_range_to_timespan::( - dt("1995-12-31T09:00:00").and_utc()..dt("1996-01-01T00:00:01").and_utc() - ), + (dt("1995-12-31T09:00:00").and_utc()..dt("1996-01-01T00:00:01").and_utc()).into() + ) + .try_into_inclusive() + .unwrap(), week_of("1995-12-31")..=week_of("1996-01-01") ); } diff --git a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/sum.rs b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/sum.rs index 399666510..c6939d1d6 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/resolutions/sum.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/resolutions/sum.rs @@ -3,21 +3,18 @@ //! Intended for "new"/"delta" charts where change of something //! is presented. -use std::{ - fmt::Debug, - marker::PhantomData, - ops::{AddAssign, Range}, -}; +use std::{fmt::Debug, marker::PhantomData, ops::AddAssign}; use blockscout_metrics_tools::AggregateTimer; use chrono::{DateTime, Utc}; use rust_decimal::prelude::Zero; -use sea_orm::{prelude::DateTimeUtc, DatabaseConnection, DbErr}; +use sea_orm::{DatabaseConnection, DbErr}; use crate::{ data_source::{DataSource, UpdateContext}, + range::UniversalRange, types::{ConsistsOf, Timespan, TimespanValue}, - UpdateError, + ChartError, }; use super::{extend_to_timespan_boundaries, reduce_each_timespan}; @@ -27,7 +24,7 @@ pub struct SumLowerResolution(PhantomData<(DS, LowerRes)>); impl DataSource for SumLowerResolution where - LowerRes: Timespan + ConsistsOf + Eq + Debug + Send, + LowerRes: Timespan + ConsistsOf + Eq + Ord + Debug + Send, HigherRes: Clone + Debug, Value: AddAssign + Zero + Send + Debug, DS: DataSource>>, @@ -49,23 +46,19 @@ where Ok(()) } - async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), UpdateError> { + async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), ChartError> { // just an adapter; inner is handled recursively Ok(()) } async fn query_data( cx: &UpdateContext<'_>, - range: Option>, + range: UniversalRange>, dependency_data_fetch_timer: &mut AggregateTimer, - ) -> Result { - let time_range_for_lower_res = range.map(extend_to_timespan_boundaries::); - let high_res_data = DS::query_data( - cx, - time_range_for_lower_res.clone(), - dependency_data_fetch_timer, - ) - .await?; + ) -> Result { + let time_range_for_lower_res = extend_to_timespan_boundaries::(range); + let high_res_data = + DS::query_data(cx, time_range_for_lower_res, dependency_data_fetch_timer).await?; Ok(reduce_each_timespan( high_res_data, |t| LowerRes::from_smaller(t.timespan.clone()), @@ -104,6 +97,8 @@ where #[cfg(test)] mod tests { + use std::sync::Arc; + use blockscout_metrics_tools::AggregateTimer; use pretty_assertions::assert_eq; @@ -111,8 +106,10 @@ mod tests { data_source::{types::BlockscoutMigrations, DataSource, UpdateContext}, gettable_const, lines::PredefinedMockSource, + range::UniversalRange, tests::point_construction::{d_v_int, dt, w_v_int}, types::timespans::{DateValue, Week}, + utils::MarkedDbConnection, MissingDatePolicy, }; @@ -135,7 +132,9 @@ mod tests { type MockSourceWeekly = SumLowerResolution; // db is not used in mock - let empty_db = sea_orm::Database::connect("sqlite::memory:").await.unwrap(); + let empty_db = MarkedDbConnection::in_memory(Arc::new( + sea_orm::Database::connect("sqlite::memory:").await.unwrap(), + )); let context = UpdateContext { db: &empty_db, @@ -145,7 +144,7 @@ mod tests { force_full: false, }; assert_eq!( - MockSource::query_data(&context, None, &mut AggregateTimer::new()) + MockSource::query_data(&context, UniversalRange::full(), &mut AggregateTimer::new()) .await .unwrap(), vec![ @@ -156,9 +155,13 @@ mod tests { ] ); assert_eq!( - MockSourceWeekly::query_data(&context, None, &mut AggregateTimer::new()) - .await - .unwrap(), + MockSourceWeekly::query_data( + &context, + UniversalRange::full(), + &mut AggregateTimer::new() + ) + .await + .unwrap(), vec![w_v_int("2024-07-08", 4), w_v_int("2024-07-22", 1239),] ); } diff --git a/stats/stats/src/data_source/kinds/data_manipulation/sum_point.rs b/stats/stats/src/data_source/kinds/data_manipulation/sum_point.rs index 6cc68f018..ba40c12f0 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/sum_point.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/sum_point.rs @@ -2,21 +2,19 @@ //! //! Sums all points from the other (vector) source. -use std::{ - marker::PhantomData, - ops::{AddAssign, Range}, -}; +use std::{marker::PhantomData, ops::AddAssign}; use blockscout_metrics_tools::AggregateTimer; use chrono::{DateTime, Utc}; use rust_decimal::prelude::Zero; -use sea_orm::{prelude::DateTimeUtc, DatabaseConnection, DbErr}; +use sea_orm::{DatabaseConnection, DbErr}; use crate::{ data_processing::sum, data_source::{source::DataSource, UpdateContext}, + range::UniversalRange, types::{Timespan, TimespanValue}, - UpdateError, + ChartError, }; /// Sum all dependency's data. @@ -51,19 +49,20 @@ where Ok(()) } - async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), UpdateError> { + async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), ChartError> { // just an adapter; inner is handled recursively Ok(()) } async fn query_data( cx: &UpdateContext<'_>, - _range: Option>, + _range: UniversalRange>, dependency_data_fetch_timer: &mut AggregateTimer, - ) -> Result { + ) -> Result { // it's possible to not request full data range and use last accurate point; // can be updated to work similarly to cumulative - let full_data = DS::query_data(cx, None, dependency_data_fetch_timer).await?; + let full_data = + DS::query_data(cx, UniversalRange::full(), dependency_data_fetch_timer).await?; tracing::debug!(points_len = full_data.len(), "calculating sum"); let zero = Value::zero(); sum::(&full_data, zero) diff --git a/stats/stats/src/data_source/kinds/local_db/mod.rs b/stats/stats/src/data_source/kinds/local_db/mod.rs index 35798f2c0..93cc00694 100644 --- a/stats/stats/src/data_source/kinds/local_db/mod.rs +++ b/stats/stats/src/data_source/kinds/local_db/mod.rs @@ -9,7 +9,7 @@ //! Charts are intended to be such persisted sources, //! because their data is directly retreived from the database (on requests). -use std::{fmt::Debug, marker::PhantomData, ops::Range, time::Duration}; +use std::{fmt::Debug, marker::PhantomData, time::Duration}; use blockscout_metrics_tools::AggregateTimer; use chrono::{DateTime, SubsecRound, Utc}; @@ -22,9 +22,9 @@ use parameters::{ }, point::PassPoint, }, - DefaultCreate, DefaultQueryLast, DefaultQueryVec, + DefaultCreate, DefaultQueryLast, DefaultQueryVec, QueryLastWithEstimationFallback, }; -use sea_orm::{prelude::DateTimeUtc, DatabaseConnection, DbErr}; +use sea_orm::{DatabaseConnection, DbErr}; use crate::{ charts::{ @@ -33,7 +33,9 @@ use crate::{ ChartProperties, Named, }, data_source::{DataSource, UpdateContext}, - metrics, UpdateError, + metrics, + range::UniversalRange, + ChartError, }; use super::auxiliary::PartialCumulative; @@ -53,7 +55,7 @@ pub mod parameters; /// /// See [module-level documentation](self) for more details. pub struct LocalDbChartSource( - PhantomData<(MainDep, ResolutionDep, Create, Update, Query, ChartProps)>, + pub PhantomData<(MainDep, ResolutionDep, Create, Update, Query, ChartProps)>, ) where MainDep: DataSource, @@ -114,6 +116,15 @@ pub type DirectPointLocalDbChartSource = LocalDbChartSource< C, >; +pub type DirectPointLocalDbChartSourceWithEstimate = LocalDbChartSource< + Dependency, + (), + DefaultCreate, + PassPoint, + QueryLastWithEstimationFallback, + C, +>; + impl LocalDbChartSource where @@ -130,8 +141,8 @@ where async fn update_itself_inner( cx: &UpdateContext<'_>, dependency_data_fetch_timer: &mut AggregateTimer, - ) -> Result<(), UpdateError> { - let metadata = get_chart_metadata(cx.db, &ChartProps::key()).await?; + ) -> Result<(), ChartError> { + let metadata = get_chart_metadata(cx.db.connection.as_ref(), &ChartProps::key()).await?; if let Some(last_updated_at) = metadata.last_updated_at { if postgres_timestamps_eq(cx.time, last_updated_at) { // no need to perform update. @@ -152,13 +163,13 @@ where } } let chart_id = metadata.id; - let min_blockscout_block = get_min_block_blockscout(cx.blockscout) + let min_blockscout_block = get_min_block_blockscout(cx.blockscout.connection.as_ref()) .await - .map_err(UpdateError::BlockscoutDB)?; + .map_err(ChartError::BlockscoutDB)?; let last_accurate_point = last_accurate_point::( chart_id, min_blockscout_block, - cx.db, + cx.db.connection.as_ref(), cx.force_full, ChartProps::approximate_trailing_points(), ChartProps::missing_date_policy(), @@ -174,7 +185,7 @@ where ) .await?; tracing::info!(chart =% ChartProps::key(), "updating chart metadata"); - Update::update_metadata(cx.db, chart_id, cx.time).await?; + Update::update_metadata(cx.db.connection.as_ref(), chart_id, cx.time).await?; Ok(()) } @@ -218,7 +229,7 @@ where Create::create(db, init_time).await } - async fn update_itself(cx: &UpdateContext<'_>) -> Result<(), UpdateError> { + async fn update_itself(cx: &UpdateContext<'_>) -> Result<(), ChartError> { // set up metrics + write some logs let mut dependency_data_fetch_timer = AggregateTimer::new(); @@ -247,11 +258,13 @@ where async fn query_data( cx: &UpdateContext<'_>, - range: Option>, + range: UniversalRange>, dependency_data_fetch_timer: &mut AggregateTimer, - ) -> Result { + ) -> Result { let _timer = dependency_data_fetch_timer.start_interval(); - Query::query_data(cx, range).await + // maybe add `fill_missing_dates` parameter to current function as well in the future + // to get rid of "Note" in the `DataSource`'s method documentation + Query::query_data(cx, range, None, false).await } } @@ -313,10 +326,10 @@ mod tests { DataSource, UpdateContext, UpdateParameters, }, gettable_const, - tests::{init_db::init_db_all, mock_blockscout::fill_mock_blockscout_data}, + tests::{init_db::init_marked_db_all, mock_blockscout::fill_mock_blockscout_data}, types::{timespans::DateValue, TimespanValue}, update_group::{SyncUpdateGroup, UpdateGroup}, - ChartProperties, Named, UpdateError, + ChartError, ChartProperties, Named, }; type WasTriggeredStorage = Arc>; @@ -355,7 +368,7 @@ mod tests { _last_accurate_point: Option>, min_blockscout_block: i64, _dependency_data_fetch_timer: &mut AggregateTimer, - ) -> Result<(), UpdateError> { + ) -> Result<(), ChartError> { Self::record_trigger().await; // insert smth for dependency to work well let data = DateValue:: { @@ -363,9 +376,9 @@ mod tests { value: "0".to_owned(), }; let value = data.active_model(chart_id, Some(min_blockscout_block)); - insert_data_many(cx.db, vec![value]) + insert_data_many(cx.db.connection.as_ref(), vec![value]) .await - .map_err(UpdateError::StatsDB)?; + .map_err(ChartError::StatsDB)?; Ok(()) } } @@ -422,10 +435,11 @@ mod tests { #[ignore = "needs database to run"] async fn update_itself_is_triggered_once_per_group() { let _ = tracing_subscriber::fmt::try_init(); - let (db, blockscout) = init_db_all("update_itself_is_triggered_once_per_group").await; + let (db, blockscout) = + init_marked_db_all("update_itself_is_triggered_once_per_group").await; let current_time = DateTime::::from_str("2023-03-01T12:00:00Z").unwrap(); let current_date = current_time.date_naive(); - fill_mock_blockscout_data(&blockscout, current_date).await; + fill_mock_blockscout_data(blockscout.connection.as_ref(), current_date).await; let enabled = HashSet::from( [TestedChartProps::key(), ChartDependedOnTestedProps::key()].map(|l| l.to_owned()), ); @@ -436,7 +450,7 @@ mod tests { .collect(); let group = SyncUpdateGroup::new(&mutexes, Arc::new(TestUpdateGroup)).unwrap(); group - .create_charts_with_mutexes(&db, Some(current_time), &enabled) + .create_charts_with_mutexes(db.connection.as_ref(), Some(current_time), &enabled) .await .unwrap(); diff --git a/stats/stats/src/data_source/kinds/local_db/parameter_traits.rs b/stats/stats/src/data_source/kinds/local_db/parameter_traits.rs index 8de6b4aa2..d62888188 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameter_traits.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameter_traits.rs @@ -1,14 +1,15 @@ -use std::{future::Future, marker::Send, ops::Range}; +use std::{future::Future, marker::Send}; use blockscout_metrics_tools::AggregateTimer; use chrono::{DateTime, Utc}; -use sea_orm::{prelude::DateTimeUtc, DatabaseConnection, DbErr}; +use sea_orm::{DatabaseConnection, DbErr}; use crate::{ charts::db_interaction::write::set_last_updated_at, data_source::{DataSource, UpdateContext}, + range::UniversalRange, types::TimespanValue, - UpdateError, + ChartError, RequestedPointsLimit, }; /// In most cases, [`super::DefaultCreate`] is enough. @@ -37,18 +38,18 @@ where last_accurate_point: Option>, min_blockscout_block: i64, dependency_data_fetch_timer: &mut AggregateTimer, - ) -> impl Future> + Send; + ) -> impl Future> + Send; /// Update only chart metadata. fn update_metadata( db: &DatabaseConnection, chart_id: i32, update_time: DateTime, - ) -> impl Future> + Send { + ) -> impl Future> + Send { async move { set_last_updated_at(chart_id, db, update_time) .await - .map_err(UpdateError::StatsDB) + .map_err(ChartError::StatsDB) } } } @@ -61,6 +62,8 @@ pub trait QueryBehaviour { /// Retrieve chart data from local storage. fn query_data( cx: &UpdateContext<'_>, - range: Option>, - ) -> impl Future> + Send; + range: UniversalRange>, + points_limit: Option, + fill_missing_dates: bool, + ) -> impl Future> + Send; } diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/mod.rs b/stats/stats/src/data_source/kinds/local_db/parameters/mod.rs index 0b712df3e..ea2c60291 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/mod.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/mod.rs @@ -3,4 +3,6 @@ mod query; pub mod update; pub use create::DefaultCreate; -pub use query::{DefaultQueryLast, DefaultQueryVec}; +pub use query::{ + DefaultQueryLast, DefaultQueryVec, QueryLastWithEstimationFallback, ValueEstimation, +}; diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/query.rs b/stats/stats/src/data_source/kinds/local_db/parameters/query.rs index ec2560b03..b39ed7a4b 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/query.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/query.rs @@ -1,14 +1,14 @@ -use std::{fmt::Debug, marker::PhantomData, ops::Range}; +use std::{fmt::Debug, marker::PhantomData}; -use sea_orm::prelude::DateTimeUtc; +use chrono::{DateTime, Utc}; use crate::{ - charts::db_interaction::read::get_counter_data, + charts::db_interaction::read::{get_counter_data, get_line_chart_data}, data_source::{kinds::local_db::parameter_traits::QueryBehaviour, UpdateContext}, - get_line_chart_data, - types::{timespans::DateValue, Timespan, TimespanValue}, - utils::exclusive_datetime_range_to_inclusive, - ChartProperties, UpdateError, + range::UniversalRange, + types::{timespans::DateValue, ExtendedTimespanValue, Timespan}, + utils::MarkedDbConnection, + ChartError, ChartProperties, RequestedPointsLimit, }; /// Usually the choice for line charts @@ -19,7 +19,7 @@ where C: ChartProperties, C::Resolution: Timespan + Ord + Debug + Clone + Send, { - type Output = Vec>; + type Output = Vec>; /// Retrieve chart data from local storage. /// @@ -28,13 +28,14 @@ where /// Expects metadata to be consistent with stored data async fn query_data( cx: &UpdateContext<'_>, - range: Option>, - ) -> Result { + range: UniversalRange>, + points_limit: Option, + fill_missing_dates: bool, + ) -> Result { // In DB we store data with date precision. Also, `get_line_chart_data` // works with inclusive range. Therefore, we need to convert the range and // get date without time. - let range = range.map(exclusive_datetime_range_to_inclusive); - let (start, end) = range.map(|r| r.into_inner()).unzip(); + let (start, end) = range.into_inclusive_pair(); // At the same time, update-time relevance for local charts // is achieved while requesting remote source data. @@ -46,21 +47,17 @@ where // same for weeks or other resolutions. let start = start.map(|s| C::Resolution::from_date(s.date_naive())); let end = end.map(|e| C::Resolution::from_date(e.date_naive())); - let values: Vec> = - get_line_chart_data::( - cx.db, - &C::name(), - start, - end, - None, - C::missing_date_policy(), - false, - C::approximate_trailing_points(), - ) - .await? - .into_iter() - .map(TimespanValue::from) - .collect(); + let values = get_line_chart_data::( + cx.db.connection.as_ref(), + &C::name(), + start, + end, + points_limit, + C::missing_date_policy(), + fill_missing_dates, + C::approximate_trailing_points(), + ) + .await?; Ok(values) } } @@ -73,19 +70,140 @@ impl QueryBehaviour for DefaultQueryLast { async fn query_data( cx: &UpdateContext<'_>, - _range: Option>, - ) -> Result { + _range: UniversalRange>, + _points_limit: Option, + _fill_missing_dates: bool, + ) -> Result { let value = get_counter_data( - cx.db, + cx.db.connection.as_ref(), &C::name(), Some(cx.time.date_naive()), C::missing_date_policy(), ) .await? - .ok_or(UpdateError::Internal(format!( + .ok_or(ChartError::Internal(format!( "no data for counter '{}' was found", C::name() )))?; Ok(value) } } + +#[trait_variant::make(Send)] +pub trait ValueEstimation { + async fn estimate(blockscout: &MarkedDbConnection) -> Result, ChartError>; +} + +pub struct QueryLastWithEstimationFallback(PhantomData<(E, C)>) +where + C: ChartProperties, + E: ValueEstimation; + +impl QueryBehaviour for QueryLastWithEstimationFallback +where + C: ChartProperties, + E: ValueEstimation, +{ + type Output = DateValue; + + async fn query_data( + cx: &UpdateContext<'_>, + _range: UniversalRange>, + _points_limit: Option, + _fill_missing_dates: bool, + ) -> Result { + let value = match get_counter_data( + cx.db.connection.as_ref(), + &C::name(), + Some(cx.time.date_naive()), + C::missing_date_policy(), + ) + .await? + { + Some(v) => v, + None => E::estimate(cx.blockscout).await?, + }; + Ok(value) + } +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use chrono::NaiveDate; + use entity::sea_orm_active_enums::ChartType; + use pretty_assertions::assert_eq; + + use super::*; + + use crate::{ + data_source::{types::BlockscoutMigrations, UpdateContext, UpdateParameters}, + tests::init_db::init_marked_db_all, + types::timespans::DateValue, + ChartError, MissingDatePolicy, Named, + }; + + #[tokio::test] + #[ignore = "needs database to run"] + async fn fallback_query_works() { + let _ = tracing_subscriber::fmt::try_init(); + let (db, blockscout) = init_marked_db_all("fallback_query_works").await; + let current_time = chrono::DateTime::from_str("2023-03-01T12:00:00Z").unwrap(); + + let parameters = UpdateParameters { + db: &db, + blockscout: &blockscout, + blockscout_applied_migrations: BlockscoutMigrations::latest(), + update_time_override: Some(current_time), + force_full: true, + }; + let cx = UpdateContext::from_params_now_or_override(parameters.clone()); + + struct TestFallback; + + fn expected_estimate() -> DateValue { + DateValue { + timespan: chrono::NaiveDate::MAX, + value: "estimate".to_string(), + } + } + + impl ValueEstimation for TestFallback { + async fn estimate( + _blockscout: &MarkedDbConnection, + ) -> Result, ChartError> { + Ok(expected_estimate()) + } + } + + pub struct InvalidProperties; + impl Named for InvalidProperties { + fn name() -> String { + "totalBlocks".into() + } + } + impl ChartProperties for InvalidProperties { + type Resolution = NaiveDate; + + fn chart_type() -> ChartType { + ChartType::Counter + } + fn missing_date_policy() -> MissingDatePolicy { + MissingDatePolicy::FillPrevious + } + } + + assert_eq!( + expected_estimate(), + QueryLastWithEstimationFallback::::query_data( + &cx, + UniversalRange::full(), + None, + true + ) + .await + .unwrap() + ); + } +} diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/mod.rs b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/mod.rs index 28548d13d..d4e83821d 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/mod.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/mod.rs @@ -17,8 +17,9 @@ use crate::{ types::Get, UpdateContext, }, - types::{Timespan, TimespanDuration, TimespanValue}, - ChartProperties, UpdateError, + range::UniversalRange, + types::{ExtendedTimespanValue, Timespan, TimespanDuration, TimespanValue}, + ChartError, ChartProperties, }; pub mod parameter_traits; @@ -39,7 +40,7 @@ where ResolutionDep: DataSource, BatchStep: BatchStepBehaviour, BatchSizeUpperBound: Get>, - Query: QueryBehaviour>>, + Query: QueryBehaviour>>, ChartProps: ChartProperties; impl @@ -50,7 +51,7 @@ where ResolutionDep: DataSource, BatchStep: BatchStepBehaviour, BatchSizeUpperBound: Get>, - Query: QueryBehaviour>>, + Query: QueryBehaviour>>, ChartProps: ChartProperties, ChartProps::Resolution: Timespan + Ord + Clone + Debug + Send, { @@ -60,7 +61,7 @@ where last_accurate_point: Option>, min_blockscout_block: i64, dependency_data_fetch_timer: &mut AggregateTimer, - ) -> Result<(), UpdateError> { + ) -> Result<(), ChartError> { let now = cx.time; let update_from = last_accurate_point .clone() @@ -68,10 +69,10 @@ where let update_range_start = match update_from { Some(d) => d, None => ChartProps::Resolution::from_date( - get_min_date_blockscout(cx.blockscout) + get_min_date_blockscout(cx.blockscout.connection.as_ref()) .await .map(|time| time.date()) - .map_err(UpdateError::BlockscoutDB)?, + .map_err(ChartError::BlockscoutDB)?, ), }; @@ -106,7 +107,12 @@ where ) .await?; // for query in `get_previous_step_last_point` to work correctly - Self::update_metadata(cx.db, chart_id, range.into_date_time_range().end).await?; + Self::update_metadata( + cx.db.connection.as_ref(), + chart_id, + range.into_date_time_range().end, + ) + .await?; let elapsed: std::time::Duration = now.elapsed(); tracing::info!( found =? found, @@ -123,20 +129,27 @@ where async fn get_previous_step_last_point( cx: &UpdateContext<'_>, this_step_start: Resolution, -) -> Result, UpdateError> +) -> Result, ChartError> where Resolution: Timespan + Clone, - Query: QueryBehaviour>>, + Query: QueryBehaviour>>, { let previous_step_last_point_timespan = this_step_start.saturating_previous_timespan(); let last_point_range_values = Query::query_data( cx, - Some(previous_step_last_point_timespan.clone().into_time_range()), + previous_step_last_point_timespan + .clone() + .into_time_range() + .into(), + None, + false, ) .await?; + // might be replaced with `fill_missing_dates=true` let previous_step_last_point = last_point_range_values .last() .cloned() + .map(|p| p.into()) // `None` means // - if `MissingDatePolicy` is `FillZero` = the value is 0 // - if `MissingDatePolicy` is `FillPrevious` = no previous value was found at this moment in time = value at the point is 0 @@ -145,7 +158,7 @@ where )); if last_point_range_values.len() > 1 { // return error because it's likely that date in `previous_step_last_point` is incorrect - return Err(UpdateError::Internal("Retrieved 2 points from previous step; probably an issue with range construction and handling".to_owned())); + return Err(ChartError::Internal("Retrieved 2 points from previous step; probably an issue with range construction and handling".to_owned())); } Ok(previous_step_last_point) } @@ -158,20 +171,20 @@ async fn batch_update_values_step last_accurate_point: TimespanValue, range: BatchRange, dependency_data_fetch_timer: &mut AggregateTimer, -) -> Result +) -> Result where MainDep: DataSource, ResolutionDep: DataSource, Resolution: Timespan, BatchStep: BatchStepBehaviour, { - let query_range = range.into_date_time_range(); + let query_range: UniversalRange<_> = range.into_date_time_range().into(); let main_data = - MainDep::query_data(cx, Some(query_range.clone()), dependency_data_fetch_timer).await?; + MainDep::query_data(cx, query_range.clone(), dependency_data_fetch_timer).await?; let resolution_data = - ResolutionDep::query_data(cx, Some(query_range), dependency_data_fetch_timer).await?; + ResolutionDep::query_data(cx, query_range, dependency_data_fetch_timer).await?; let found = BatchStep::batch_update_values_step_with( - cx.db, + cx.db.connection.as_ref(), chart_id, cx.time, min_blockscout_block, @@ -222,12 +235,12 @@ fn generate_batch_ranges( start: Resolution, end: DateTime, max_step: TimespanDuration, -) -> Result>, UpdateError> +) -> Result>, ChartError> where Resolution: Timespan + Ord + Clone, { if max_step.repeats() == 0 { - return Err(UpdateError::Internal( + return Err(ChartError::Internal( "Zero maximum batch step is not allowed".into(), )); } @@ -378,12 +391,15 @@ mod tests { use crate::{ data_source::{ - kinds::local_db::{ - parameters::{ - update::batching::{parameters::mock::RecordingPassStep, BatchUpdate}, - DefaultCreate, DefaultQueryVec, + kinds::{ + data_manipulation::map::StripExt, + local_db::{ + parameters::{ + update::batching::{parameters::mock::RecordingPassStep, BatchUpdate}, + DefaultCreate, DefaultQueryVec, + }, + LocalDbChartSource, }, - LocalDbChartSource, }, types::Get, }, @@ -432,7 +448,7 @@ mod tests { } type ThisRecordingBatchUpdate = BatchUpdate< - AccountsGrowth, + StripExt, (), ThisRecordingStep, Batch1Day, @@ -441,7 +457,7 @@ mod tests { >; type RecordingChart = LocalDbChartSource< - AccountsGrowth, + StripExt, (), DefaultCreate, ThisRecordingBatchUpdate, diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameter_traits.rs b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameter_traits.rs index d2277ccb5..d4150c3c9 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameter_traits.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameter_traits.rs @@ -5,7 +5,7 @@ use sea_orm::DatabaseConnection; use crate::{ types::{Timespan, TimespanValue}, - UpdateError, + ChartError, }; pub trait BatchStepBehaviour @@ -25,5 +25,5 @@ where last_accurate_point: TimespanValue, main_data: MainInput, resolution_data: ResolutionInput, - ) -> impl Future> + std::marker::Send; + ) -> impl Future> + std::marker::Send; } diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/cumulative.rs b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/cumulative.rs index f8242e729..ffaad7478 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/cumulative.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/cumulative.rs @@ -9,7 +9,7 @@ use sea_orm::DatabaseConnection; use crate::{ data_source::kinds::local_db::parameters::update::batching::parameter_traits::BatchStepBehaviour, types::{Timespan, TimespanValue}, - ChartProperties, UpdateError, + ChartError, ChartProperties, }; use super::PassVecStep; @@ -37,9 +37,9 @@ where last_accurate_point: TimespanValue, main_data: Vec>, _resolution_data: (), - ) -> Result { + ) -> Result { let partial_sum = last_accurate_point.value.parse::().map_err(|e| { - UpdateError::Internal(format!( + ChartError::Internal(format!( "failed to parse value in chart '{}': {e}", ChartProps::key() )) diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/mock.rs b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/mock.rs index 846acfd52..0d0cc61ec 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/mock.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/mock.rs @@ -5,7 +5,7 @@ use sea_orm::DatabaseConnection; use crate::{ data_source::kinds::local_db::parameters::update::batching::parameter_traits::BatchStepBehaviour, - tests::recorder::Recorder, types::timespans::DateValue, UpdateError, + tests::recorder::Recorder, types::timespans::DateValue, ChartError, }; use super::PassVecStep; @@ -40,7 +40,7 @@ where last_accurate_point: DateValue, main_data: Vec>, resolution_data: (), - ) -> Result { + ) -> Result { StepsRecorder::record(StepInput { chart_id, update_time, diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/mod.rs b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/mod.rs index ce27c16d3..41e274fd8 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/mod.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/update/batching/parameters/mod.rs @@ -8,7 +8,7 @@ use crate::{ timespans::{Month, Week, Year}, Timespan, TimespanDuration, TimespanValue, }, - UpdateError, + ChartError, }; use super::parameter_traits::BatchStepBehaviour; @@ -42,7 +42,7 @@ where _last_accurate_point: TimespanValue, main_data: Vec>, _resolution_data: (), - ) -> Result { + ) -> Result { let found = main_data.len(); // note: right away cloning another chart will not result in exact copy, // because if the other chart is `FillPrevious`, then omitted starting point @@ -56,7 +56,7 @@ where .map(|value| value.active_model(chart_id, Some(min_blockscout_block))); insert_data_many(db, values) .await - .map_err(UpdateError::StatsDB)?; + .map_err(ChartError::StatsDB)?; Ok(found) } } diff --git a/stats/stats/src/data_source/kinds/local_db/parameters/update/point.rs b/stats/stats/src/data_source/kinds/local_db/parameters/update/point.rs index 1bba427a6..f3b3c4a16 100644 --- a/stats/stats/src/data_source/kinds/local_db/parameters/update/point.rs +++ b/stats/stats/src/data_source/kinds/local_db/parameters/update/point.rs @@ -5,8 +5,9 @@ use blockscout_metrics_tools::AggregateTimer; use crate::{ charts::db_interaction::write::insert_data_many, data_source::{kinds::local_db::UpdateBehaviour, DataSource, UpdateContext}, + range::UniversalRange, types::{Timespan, TimespanValue}, - UpdateError, + ChartError, }; /// Store output of the `MainDep` right in the local db @@ -23,13 +24,13 @@ where _last_accurate_point: Option>, min_blockscout_block: i64, remote_fetch_timer: &mut AggregateTimer, - ) -> Result<(), UpdateError> { + ) -> Result<(), ChartError> { // range doesn't make sense there; thus is not used - let data = MainDep::query_data(cx, None, remote_fetch_timer).await?; + let data = MainDep::query_data(cx, UniversalRange::full(), remote_fetch_timer).await?; let value = data.active_model(chart_id, Some(min_blockscout_block)); - insert_data_many(cx.db, vec![value]) + insert_data_many(cx.db.connection.as_ref(), vec![value]) .await - .map_err(UpdateError::StatsDB)?; + .map_err(ChartError::StatsDB)?; Ok(()) } } diff --git a/stats/stats/src/data_source/kinds/remote_db/mod.rs b/stats/stats/src/data_source/kinds/remote_db/mod.rs index 0a68f44c5..5a47180d4 100644 --- a/stats/stats/src/data_source/kinds/remote_db/mod.rs +++ b/stats/stats/src/data_source/kinds/remote_db/mod.rs @@ -21,7 +21,6 @@ mod query; use std::{ future::Future, marker::{PhantomData, Send}, - ops::Range, }; use blockscout_metrics_tools::AggregateTimer; @@ -30,7 +29,8 @@ use sea_orm::{DatabaseConnection, DbErr}; use crate::{ data_source::{source::DataSource, types::UpdateContext}, - UpdateError, + range::UniversalRange, + ChartError, }; pub use query::{ @@ -47,8 +47,8 @@ pub trait RemoteQueryBehaviour { /// Retrieve chart data from remote storage. fn query_data( cx: &UpdateContext<'_>, - range: Option>>, - ) -> impl Future> + Send; + range: UniversalRange>, + ) -> impl Future> + Send; } impl DataSource for RemoteDatabaseSource { @@ -69,14 +69,14 @@ impl DataSource for RemoteDatabaseSource { async fn query_data( cx: &UpdateContext<'_>, - range: Option>>, + range: UniversalRange>, remote_fetch_timer: &mut AggregateTimer, - ) -> Result<::Output, UpdateError> { + ) -> Result<::Output, ChartError> { let _interval = remote_fetch_timer.start_interval(); Q::query_data(cx, range).await } - async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), UpdateError> { + async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), ChartError> { Ok(()) } } diff --git a/stats/stats/src/data_source/kinds/remote_db/query/all.rs b/stats/stats/src/data_source/kinds/remote_db/query/all.rs index d7fdd777a..0583240e4 100644 --- a/stats/stats/src/data_source/kinds/remote_db/query/all.rs +++ b/stats/stats/src/data_source/kinds/remote_db/query/all.rs @@ -3,21 +3,23 @@ use std::{ ops::Range, }; -use sea_orm::{prelude::DateTimeUtc, FromQueryResult, Statement}; +use chrono::{DateTime, Utc}; +use sea_orm::{FromQueryResult, Statement}; use crate::{ data_source::{ kinds::remote_db::RemoteQueryBehaviour, types::{BlockscoutMigrations, UpdateContext}, }, + range::{data_source_query_range_to_db_statement_range, UniversalRange}, types::TimespanValue, - UpdateError, + ChartError, }; pub trait StatementFromRange { /// `completed_migrations` can be used for selecting more optimal query fn get_statement( - range: Option>, + range: Option>>, completed_migrations: &BlockscoutMigrations, ) -> Statement; } @@ -28,31 +30,39 @@ pub trait StatementFromRange { /// `P` - Type of point to retrieve within query. /// `DateValue` can be used to avoid parsing the values, /// but `DateValue` or other types can be useful sometimes. -pub struct PullAllWithAndSort(PhantomData<(S, Resolution, Value)>) +pub struct PullAllWithAndSort( + PhantomData<(S, Resolution, Value, AllRangeSource)>, +) where S: StatementFromRange, Resolution: Ord + Send, Value: Send, - TimespanValue: FromQueryResult; + TimespanValue: FromQueryResult, + AllRangeSource: RemoteQueryBehaviour>>; -impl RemoteQueryBehaviour for PullAllWithAndSort +impl RemoteQueryBehaviour + for PullAllWithAndSort where S: StatementFromRange, Resolution: Ord + Send, Value: Send, TimespanValue: FromQueryResult, + AllRangeSource: RemoteQueryBehaviour>>, { type Output = Vec>; async fn query_data( cx: &UpdateContext<'_>, - range: Option>, - ) -> Result>, UpdateError> { - let query = S::get_statement(range, &cx.blockscout_applied_migrations); + range: UniversalRange>, + ) -> Result>, ChartError> { + // to not overcomplicate the queries + let query_range = + data_source_query_range_to_db_statement_range::(cx, range).await?; + let query = S::get_statement(query_range, &cx.blockscout_applied_migrations); let mut data = TimespanValue::::find_by_statement(query) - .all(cx.blockscout) + .all(cx.blockscout.connection.as_ref()) .await - .map_err(UpdateError::BlockscoutDB)?; + .map_err(ChartError::BlockscoutDB)?; // linear time for sorted sequences data.sort_unstable_by(|a, b| a.timespan.cmp(&b.timespan)); // can't use sort_*_by_key: https://github.com/rust-lang/rust/issues/34162 diff --git a/stats/stats/src/data_source/kinds/remote_db/query/each.rs b/stats/stats/src/data_source/kinds/remote_db/query/each.rs index 76285aaa3..ea8dcf397 100644 --- a/stats/stats/src/data_source/kinds/remote_db/query/each.rs +++ b/stats/stats/src/data_source/kinds/remote_db/query/each.rs @@ -12,9 +12,9 @@ use crate::{ kinds::remote_db::RemoteQueryBehaviour, types::{BlockscoutMigrations, UpdateContext}, }, - exclusive_datetime_range_to_inclusive, + range::{exclusive_range_to_inclusive, UniversalRange}, types::{Timespan, TimespanValue}, - UpdateError, + ChartError, }; pub trait StatementFromTimespan { @@ -50,21 +50,22 @@ where async fn query_data( cx: &UpdateContext<'_>, - range: Option>>, - ) -> Result>, UpdateError> { - let query_range = if let Some(r) = range { + range: UniversalRange>, + ) -> Result>, ChartError> { + let query_range = if let Some(r) = range.clone().try_into_exclusive() { r } else { - AllRangeSource::query_data(cx, None).await? + let whole_range = AllRangeSource::query_data(cx, UniversalRange::full()).await?; + range.into_exclusive_with_backup(whole_range) }; let points = split_time_range_into_resolution_points::(query_range); let mut collected_data = Vec::with_capacity(points.len()); for point_range in points { let query = S::get_statement(point_range.clone(), &cx.blockscout_applied_migrations); let point_value = ValueWrapper::::find_by_statement(query) - .one(cx.blockscout) + .one(cx.blockscout.connection.as_ref()) .await - .map_err(UpdateError::BlockscoutDB)?; + .map_err(ChartError::BlockscoutDB)?; if let Some(ValueWrapper { value }) = point_value { let timespan = resolution_from_range(point_range); collected_data.push(TimespanValue { timespan, value }); @@ -75,7 +76,7 @@ where } fn resolution_from_range(range: Range>) -> R { - let range = exclusive_datetime_range_to_inclusive(range); + let range = exclusive_range_to_inclusive(range); let res = R::from_date(range.start().date_naive()); let res_verify = R::from_date(range.end().date_naive()); if res_verify != res { diff --git a/stats/stats/src/data_source/kinds/remote_db/query/one.rs b/stats/stats/src/data_source/kinds/remote_db/query/one.rs index 54fa365fb..1a1dcedc2 100644 --- a/stats/stats/src/data_source/kinds/remote_db/query/one.rs +++ b/stats/stats/src/data_source/kinds/remote_db/query/one.rs @@ -1,17 +1,16 @@ -use std::{ - marker::{PhantomData, Send}, - ops::Range, -}; +use std::marker::{PhantomData, Send}; -use sea_orm::{prelude::DateTimeUtc, FromQueryResult, Statement}; +use chrono::{DateTime, Utc}; +use sea_orm::{FromQueryResult, Statement}; use crate::{ data_source::{ kinds::remote_db::RemoteQueryBehaviour, types::{BlockscoutMigrations, UpdateContext}, }, + range::UniversalRange, types::TimespanValue, - UpdateError, + ChartError, }; pub trait StatementForOne { @@ -42,14 +41,14 @@ where async fn query_data( cx: &UpdateContext<'_>, - _range: Option>, - ) -> Result, UpdateError> { + _range: UniversalRange>, + ) -> Result, ChartError> { let query = S::get_statement(&cx.blockscout_applied_migrations); let data = TimespanValue::::find_by_statement(query) - .one(cx.blockscout) + .one(cx.blockscout.connection.as_ref()) .await - .map_err(UpdateError::BlockscoutDB)? - .ok_or_else(|| UpdateError::Internal("query returned nothing".into()))?; + .map_err(ChartError::BlockscoutDB)? + .ok_or_else(|| ChartError::Internal("query returned nothing".into()))?; Ok(data) } } diff --git a/stats/stats/src/data_source/source.rs b/stats/stats/src/data_source/source.rs index 2f3fbb234..46507c9cf 100644 --- a/stats/stats/src/data_source/source.rs +++ b/stats/stats/src/data_source/source.rs @@ -1,13 +1,13 @@ -use std::{collections::HashSet, future::Future, marker::Send, ops::Range}; +use std::{collections::HashSet, future::Future, marker::Send}; use blockscout_metrics_tools::AggregateTimer; -use chrono::Utc; +use chrono::{DateTime, Utc}; use futures::{future::BoxFuture, FutureExt}; -use sea_orm::{prelude::DateTimeUtc, DatabaseConnection, DbErr}; +use sea_orm::{DatabaseConnection, DbErr}; use tracing::instrument; use tynm::type_name; -use crate::UpdateError; +use crate::{range::UniversalRange, ChartError}; use super::types::UpdateContext; @@ -116,7 +116,7 @@ pub trait DataSource { #[instrument(skip_all, level = tracing::Level::DEBUG, fields(source_mutex_id = Self::mutex_id()))] fn update_recursively( cx: &UpdateContext<'_>, - ) -> impl Future> + Send { + ) -> impl Future> + Send { async move { // Couldn't figure out how to control level per-argument basis (in instrumentation) // so this event is used insted, since the name is usually quite verbose @@ -146,9 +146,8 @@ pub trait DataSource { /// /// ## Description /// Update only thise data source's data (values + metadat) - fn update_itself( - cx: &UpdateContext<'_>, - ) -> impl Future> + Send; + fn update_itself(cx: &UpdateContext<'_>) + -> impl Future> + Send; /// Retrieve chart data. /// If `range` is `Some`, should return data within the range. Otherwise - all data. @@ -163,9 +162,9 @@ pub trait DataSource { /// to call [`DataSource::update_recursively`] beforehand. fn query_data( cx: &UpdateContext<'_>, - range: Option>, + range: UniversalRange>, dependency_data_fetch_timer: &mut AggregateTimer, - ) -> impl Future> + Send; + ) -> impl Future> + Send; } // Base case for recursive type @@ -197,20 +196,20 @@ impl DataSource for () { HashSet::new() } - async fn update_recursively(_cx: &UpdateContext<'_>) -> Result<(), UpdateError> { + async fn update_recursively(_cx: &UpdateContext<'_>) -> Result<(), ChartError> { // stop recursion Ok(()) } - async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), UpdateError> { + async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), ChartError> { unreachable!("not called by `update_recursively` and must not be called by anything else") } async fn query_data( _cx: &UpdateContext<'_>, - _range: Option>, + _range: UniversalRange>, _remote_fetch_timer: &mut AggregateTimer, - ) -> Result { + ) -> Result { Ok(()) } } @@ -234,14 +233,14 @@ macro_rules! impl_data_source_for_tuple { None } - async fn update_recursively(cx: &UpdateContext<'_>) -> Result<(), UpdateError> { + async fn update_recursively(cx: &UpdateContext<'_>) -> Result<(), ChartError> { $( $element_generic_name::update_recursively(cx).await?; )+ Ok(()) } - async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), UpdateError> { + async fn update_itself(_cx: &UpdateContext<'_>) -> Result<(), ChartError> { // dependencies are called in `update_recursively` // the tuple itself does not need any init Ok(()) @@ -249,9 +248,9 @@ macro_rules! impl_data_source_for_tuple { async fn query_data( cx: &UpdateContext<'_>, - range: Option>, + range: UniversalRange>, remote_fetch_timer: &mut AggregateTimer, - ) -> Result { + ) -> Result { Ok(( $( $element_generic_name::query_data(cx, range.clone(), remote_fetch_timer).await? diff --git a/stats/stats/src/data_source/tests.rs b/stats/stats/src/data_source/tests.rs index d224c9764..f48419e20 100644 --- a/stats/stats/src/data_source/tests.rs +++ b/stats/stats/src/data_source/tests.rs @@ -2,12 +2,15 @@ use std::{collections::HashSet, ops::Range, str::FromStr, sync::Arc}; use chrono::{DateTime, NaiveDate, Utc}; use entity::sea_orm_active_enums::ChartType; -use sea_orm::{prelude::*, DbBackend, Statement}; +use sea_orm::{DatabaseConnection, DbBackend, Statement}; use tokio::sync::Mutex; use super::{ kinds::{ - data_manipulation::{map::MapParseTo, resolutions::last_value::LastValueLowerResolution}, + data_manipulation::{ + map::{MapParseTo, StripExt}, + resolutions::last_value::LastValueLowerResolution, + }, local_db::{ parameters::{ update::batching::{ @@ -24,24 +27,25 @@ use super::{ types::UpdateParameters, }; use crate::{ + charts::db_interaction::read::QueryAllBlockTimestampRange, construct_update_group, data_source::{ kinds::local_db::parameters::update::batching::parameters::PassVecStep, types::BlockscoutMigrations, }, define_and_impl_resolution_properties, - tests::{init_db::init_db_all, mock_blockscout::fill_mock_blockscout_data}, + tests::{init_db::init_marked_db_all, mock_blockscout::fill_mock_blockscout_data}, types::timespans::{DateValue, Month, Week, Year}, update_group::{SyncUpdateGroup, UpdateGroup}, utils::{produce_filter_and_values, sql_with_range_filter_opt}, - ChartProperties, MissingDatePolicy, Named, UpdateError, + ChartError, ChartProperties, MissingDatePolicy, Named, }; pub struct NewContractsQuery; impl StatementFromRange for NewContractsQuery { fn get_statement( - range: Option>, + range: Option>>, completed_migrations: &BlockscoutMigrations, ) -> Statement { // choose the statement based on migration progress @@ -124,8 +128,9 @@ impl StatementFromRange for NewContractsQuery { } } -pub type NewContractsRemote = - RemoteDatabaseSource>; +pub type NewContractsRemote = RemoteDatabaseSource< + PullAllWithAndSort, +>; pub struct NewContractsChartProperties; @@ -147,7 +152,7 @@ impl ChartProperties for NewContractsChartProperties { pub type NewContracts = DirectVecLocalDbChartSource; -pub type NewContractsInt = MapParseTo; +pub type NewContractsInt = MapParseTo, i64>; pub struct ContractsGrowthProperties; @@ -180,18 +185,19 @@ define_and_impl_resolution_properties!( pub type ContractsGrowth = DailyCumulativeLocalDbChartSource; +type ContractsGrowthS = StripExt; pub type ContractsGrowthWeekly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch30Weeks, ContractsGrowthWeeklyProperties, >; pub type ContractsGrowthMonthly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch36Months, ContractsGrowthMonthlyProperties, >; pub type ContractsGrowthYearly = DirectVecLocalDbChartSource< - LastValueLowerResolution, + LastValueLowerResolution, Batch30Years, ContractsGrowthYearlyProperties, >; @@ -211,7 +217,7 @@ impl BatchStepBehaviour>, ()> _last_accurate_point: DateValue, _main_data: Vec>, _resolution_data: (), - ) -> Result { + ) -> Result { // do something (just an example, not intended for running) todo!(); // save data @@ -261,10 +267,10 @@ construct_update_group!(ExampleUpdateGroup { #[ignore = "needs database to run"] async fn update_examples() { let _ = tracing_subscriber::fmt::try_init(); - let (db, blockscout) = init_db_all("update_examples").await; + let (db, blockscout) = init_marked_db_all("update_examples").await; let current_time = DateTime::::from_str("2023-03-01T12:00:00Z").unwrap(); let current_date = current_time.date_naive(); - fill_mock_blockscout_data(&blockscout, current_date).await; + fill_mock_blockscout_data(blockscout.connection.as_ref(), current_date).await; let enabled = HashSet::from( [ NewContractsChartProperties::key(), @@ -284,7 +290,7 @@ async fn update_examples() { .collect(); let group = SyncUpdateGroup::new(&mutexes, Arc::new(ExampleUpdateGroup)).unwrap(); group - .create_charts_with_mutexes(&db, None, &enabled) + .create_charts_with_mutexes(db.connection.as_ref(), None, &enabled) .await .unwrap(); diff --git a/stats/stats/src/data_source/types.rs b/stats/stats/src/data_source/types.rs index 56e3b3486..58dc2bdd3 100644 --- a/stats/stats/src/data_source/types.rs +++ b/stats/stats/src/data_source/types.rs @@ -3,10 +3,12 @@ use chrono::Utc; use sea_orm::{DatabaseConnection, DbErr, EntityTrait, FromQueryResult, QueryOrder, Statement}; use tracing::warn; +use crate::utils::MarkedDbConnection; + #[derive(Clone)] pub struct UpdateParameters<'a> { - pub db: &'a DatabaseConnection, - pub blockscout: &'a DatabaseConnection, + pub db: &'a MarkedDbConnection, + pub blockscout: &'a MarkedDbConnection, pub blockscout_applied_migrations: BlockscoutMigrations, /// If `None`, it will be measured at the start of update /// (i.e. after taking mutexes) @@ -17,8 +19,8 @@ pub struct UpdateParameters<'a> { #[derive(Clone)] pub struct UpdateContext<'a> { - pub db: &'a DatabaseConnection, - pub blockscout: &'a DatabaseConnection, + pub db: &'a MarkedDbConnection, + pub blockscout: &'a MarkedDbConnection, pub blockscout_applied_migrations: BlockscoutMigrations, /// Update time pub time: chrono::DateTime, diff --git a/stats/stats/src/lib.rs b/stats/stats/src/lib.rs index 6fd73cc4a..27fb5a1ac 100644 --- a/stats/stats/src/lib.rs +++ b/stats/stats/src/lib.rs @@ -1,11 +1,14 @@ +#![recursion_limit = "256"] + mod charts; pub mod data_processing; pub mod data_source; pub mod metrics; mod missing_date; +pub mod range; pub mod update_group; pub mod update_groups; -pub(crate) mod utils; +pub mod utils; #[cfg(any(feature = "test-utils", test))] pub mod tests; @@ -16,10 +19,8 @@ pub use migration; pub use charts::{ counters, db_interaction::read::{ - get_line_chart_data, get_raw_counters, ApproxUnsignedDiff, ReadError, RequestedPointsLimit, + ApproxUnsignedDiff, QueryAllBlockTimestampRange, ReadError, RequestedPointsLimit, }, - lines, types, ChartKey, ChartProperties, ChartPropertiesObject, MissingDatePolicy, Named, - ResolutionKind, UpdateError, + lines, query_dispatch, types, ChartError, ChartKey, ChartObject, ChartProperties, + ChartPropertiesObject, MissingDatePolicy, Named, ResolutionKind, }; - -pub use utils::exclusive_datetime_range_to_inclusive; diff --git a/stats/stats/src/range.rs b/stats/stats/src/range.rs new file mode 100644 index 000000000..56b62790b --- /dev/null +++ b/stats/stats/src/range.rs @@ -0,0 +1,506 @@ +use std::ops::{Bound, Range, RangeBounds, RangeInclusive}; + +use chrono::{DateTime, Utc}; + +use crate::{ + data_source::{kinds::remote_db::RemoteQueryBehaviour, UpdateContext}, + types::{Timespan, TimespanDuration}, + ChartError, +}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UniversalRange { + /// Always inclusive, if present + pub start: Option, + pub end: Bound, +} + +impl UniversalRange { + pub fn full() -> Self { + Self { + start: None, + end: Bound::Unbounded, + } + } + + pub fn with_replaced_unbounded(self, replacement_source: Range) -> Self { + let start = match self.start { + Some(s) => Some(s), + None => Some(replacement_source.start), + }; + let end = match self.end { + Bound::Unbounded => Bound::Excluded(replacement_source.end), + _ => self.end, + }; + Self { start, end } + } + + pub fn map(self, f: impl Fn(Idx) -> T) -> UniversalRange { + UniversalRange { + start: self.start.map(&f), + end: self.end.map(f), + } + } +} + +impl From> for UniversalRange { + fn from(value: Range) -> Self { + Self { + start: Some(value.start), + end: Bound::Excluded(value.end), + } + } +} + +impl From>> for UniversalRange { + fn from(value: Range>) -> Self { + Self { + start: value.start, + end: match value.end { + Some(e) => Bound::Excluded(e), + None => Bound::Unbounded, + }, + } + } +} + +impl From> for UniversalRange { + fn from(value: RangeInclusive) -> Self { + let (start, end) = value.into_inner(); + Self { + start: Some(start), + end: Bound::Included(end), + } + } +} + +impl From>> for UniversalRange { + fn from(value: RangeInclusive>) -> Self { + let (start, end) = value.into_inner(); + Self { + start, + end: match end { + Some(e) => Bound::Included(e), + None => Bound::Unbounded, + }, + } + } +} + +impl UniversalRange { + /// None only if no backup is provided + fn into_exclusive_inner(self, backup_bounds: Option>) -> Option> { + let (backup_start, backup_end) = backup_bounds.map(|b| (b.start, b.end)).unzip(); + let start = self.start.or(backup_start)?; + match self.end { + Bound::Included(end) => Some(inclusive_range_to_exclusive(start..=end)), + Bound::Excluded(end) => Some(start..end), + Bound::Unbounded => { + let end = backup_end?; + Some(start..end) + } + } + } + + /// `None` if any of the ends is unbounded + pub fn try_into_exclusive(self) -> Option> { + self.into_exclusive_inner(None) + } + + /// Take bounds from `backup_bounds` if any of them is unbounded. + pub fn into_exclusive_with_backup(self, backup_bounds: Range) -> Range { + self.into_exclusive_inner(Some(backup_bounds)) + .expect("`backup_bounds` is not None") + } +} + +impl UniversalRange { + fn into_inclusive_inner( + self, + backup_bounds: Option>, + ) -> Option> { + let (backup_start, backup_end) = backup_bounds.map(|b| b.into_inner()).unzip(); + let start = self.start.or(backup_start)?; + match self.end { + Bound::Included(end) => Some(start..=end), + Bound::Excluded(end) => Some(exclusive_range_to_inclusive(start..end)), + Bound::Unbounded => { + let end = backup_end?; + Some(start..=end) + } + } + } + + /// `None` if any of the ends is unbounded + pub fn try_into_inclusive(self) -> Option> { + self.into_inclusive_inner(None) + } + + /// Take bounds from `backup_bounds` if any of them is unbounded. + pub fn into_inclusive_with_backup( + self, + backup_bounds: RangeInclusive, + ) -> RangeInclusive { + self.into_inclusive_inner(Some(backup_bounds)) + .expect("`backup_bounds` is not None") + } + + pub fn into_inclusive_pair(self) -> (Option, Option) { + match (self.start, self.end) { + (start, Bound::Unbounded) => (start, None), + (start, Bound::Included(e)) => (start, Some(e)), + (None, Bound::Excluded(e)) => (None, Some(e.saturating_dec())), + (Some(s), Bound::Excluded(e)) => { + Some(exclusive_range_to_inclusive(s..e).into_inner()).unzip() + } + } + } +} + +impl> RangeBounds for UniversalRange { + fn start_bound(&self) -> Bound<&Idx> { + match &self.start { + Some(s) => Bound::Included(s), + None => Bound::Unbounded, + } + } + + fn end_bound(&self) -> Bound<&Idx> { + self.end.as_ref() + } +} + +pub fn exclusive_range_to_inclusive( + r: Range, +) -> RangeInclusive { + let mut start = r.start; + let end = match r.end.checked_dec() { + Some(new_end) => new_end, + None => { + // current end is the minimum value and is excluded, thus + // `self` range is empty + // so we need to produce an empty range as well + if start.checked_dec().is_none() { + // will not produce an empty range iff there is only + // one value of `Idx` type, which will not be encountered + // in practice + start = start.saturating_inc(); + } + r.end + } + }; + start..=end +} + +pub fn inclusive_range_to_exclusive(r: RangeInclusive) -> Range { + let (start, end) = r.into_inner(); + // impossible to include max value in exclusive range, + // so we leave it excluded in such case + start..end.saturating_inc() +} + +pub trait Incrementable { + fn saturating_inc(&self) -> Self; + fn checked_inc(&self) -> Option + where + Self: Sized; +} + +impl Incrementable for T { + fn saturating_inc(&self) -> Self { + self.saturating_next_timespan() + } + + fn checked_inc(&self) -> Option + where + Self: Sized, + { + self.checked_add(TimespanDuration::from_timespan_repeats(1)) + } +} + +impl Incrementable for DateTime { + fn saturating_inc(&self) -> Self { + self.checked_inc().unwrap_or(DateTime::::MAX_UTC) + } + + fn checked_inc(&self) -> Option { + self.checked_add_signed(chrono::Duration::nanoseconds(1)) + } +} + +// for tests +impl Incrementable for i32 { + fn saturating_inc(&self) -> Self { + self.saturating_add(1) + } + + fn checked_inc(&self) -> Option { + self.checked_add(1) + } +} + +pub trait Decrementable { + fn saturating_dec(&self) -> Self; + fn checked_dec(&self) -> Option + where + Self: Sized; +} + +impl Decrementable for T { + fn saturating_dec(&self) -> Self { + self.saturating_previous_timespan() + } + + fn checked_dec(&self) -> Option + where + Self: Sized, + { + self.checked_sub(TimespanDuration::from_timespan_repeats(1)) + } +} + +impl Decrementable for DateTime { + fn saturating_dec(&self) -> Self { + self.checked_dec().unwrap_or(DateTime::::MIN_UTC) + } + + fn checked_dec(&self) -> Option { + self.checked_sub_signed(chrono::Duration::nanoseconds(1)) + } +} + +impl Decrementable for i32 { + fn saturating_dec(&self) -> Self { + self.saturating_sub(1) + } + + fn checked_dec(&self) -> Option { + self.checked_sub(1) + } +} + +// todo (future releases/features): implement one-sided range support +// for db statements (in `filter` part) +/// DB statements have a regular range to keep their implementation simpler +/// but data sources have flexible bound setting. This function converts +/// from one to another while ensuring that non-range queries stay the same. +/// +/// At the same time, it adds support for one-side unbounded ranges by +/// quering the provided all range source in such case. +pub async fn data_source_query_range_to_db_statement_range( + cx: &UpdateContext<'_>, + data_source_range: UniversalRange>, +) -> Result>>, ChartError> +where + AllRangeSource: RemoteQueryBehaviour>>, +{ + let range = if let Some(r) = data_source_range.clone().try_into_exclusive() { + Some(r) + } else if data_source_range.start.is_none() && data_source_range.end == Bound::Unbounded { + None + } else { + let whole_range = AllRangeSource::query_data(cx, UniversalRange::full()).await?; + Some(data_source_range.into_exclusive_with_backup(whole_range)) + }; + Ok(range) +} +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::point_construction::*; + use chrono::NaiveDate; + use pretty_assertions::assert_eq; + + #[test] + fn test_with_replaced_unbounded() { + let range = UniversalRange { + start: None, + end: Bound::Unbounded, + }; + let replacement = dt("2023-01-01T00:00:00")..dt("2023-01-02T00:00:00"); + let result = range.with_replaced_unbounded(replacement); + assert_eq!(result.start, Some(dt("2023-01-01T00:00:00"))); + assert_eq!(result.end, Bound::Excluded(dt("2023-01-02T00:00:00"))); + + let partial_range = UniversalRange { + start: Some(dt("2023-01-01T12:00:00")), + end: Bound::Unbounded, + }; + let result = partial_range + .with_replaced_unbounded(dt("2023-01-01T00:00:00")..dt("2023-01-02T00:00:00")); + assert_eq!(result.start, Some(dt("2023-01-01T12:00:00"))); + assert_eq!(result.end, Bound::Excluded(dt("2023-01-02T00:00:00"))); + } + + #[test] + fn test_conversion_functions() { + // Test with weeks + let exclusive = d("2023-01-01")..d("2023-01-15"); + let inclusive = exclusive_range_to_inclusive(exclusive); + assert_eq!(inclusive, d("2023-01-01")..=d("2023-01-14")); + + let inclusive = d("2023-01-01")..=d("2023-01-14"); + let exclusive = inclusive_range_to_exclusive(inclusive); + assert_eq!(exclusive, d("2023-01-01")..d("2023-01-15")); + + // Test with months + let exclusive = month_of("2023-01-01")..month_of("2023-03-01"); + let inclusive = exclusive_range_to_inclusive(exclusive); + assert_eq!(inclusive, month_of("2023-01-01")..=month_of("2023-02-01")); + } + + #[test] + fn test_into_exclusive_conversions() { + let range: UniversalRange = (d("2023-01-01")..d("2023-01-15")).into(); + assert_eq!( + range.clone().try_into_exclusive(), + Some(d("2023-01-01")..d("2023-01-15")) + ); + + let inclusive: UniversalRange = (d("2023-01-01")..=d("2023-01-14")).into(); + assert_eq!( + inclusive.clone().try_into_exclusive(), + Some(d("2023-01-01")..d("2023-01-15")) + ); + + let partial: UniversalRange = UniversalRange { + start: None, + end: Bound::Excluded(d("2023-01-15")), + }; + assert_eq!(partial.clone().try_into_exclusive(), None); + assert_eq!( + partial + .clone() + .into_exclusive_with_backup(d("2023-01-01")..d("2023-01-16")), + d("2023-01-01")..d("2023-01-15") + ); + assert_eq!( + UniversalRange { + start: Some(d("2023-01-01")), + end: Bound::Unbounded, + } + .into_exclusive_with_backup(d("2023-01-02")..d("2023-01-16")), + d("2023-01-01")..d("2023-01-16") + ); + + let unbounded = UniversalRange::full(); + assert_eq!(unbounded.clone().try_into_exclusive(), None); + + let with_backup = unbounded.into_exclusive_with_backup(d("2023-01-01")..d("2023-01-15")); + assert_eq!(with_backup, d("2023-01-01")..d("2023-01-15")); + } + + #[test] + fn test_into_inclusive_conversions() { + let range: UniversalRange = (d("2023-01-01")..=d("2023-01-15")).into(); + assert_eq!( + range.clone().try_into_inclusive(), + Some(d("2023-01-01")..=d("2023-01-15")) + ); + + let exclusive: UniversalRange = (d("2023-01-01")..d("2023-01-15")).into(); + assert_eq!( + exclusive.clone().try_into_inclusive(), + Some(d("2023-01-01")..=d("2023-01-14")) + ); + + let unbounded = UniversalRange::full(); + assert_eq!(unbounded.clone().try_into_inclusive(), None); + + let with_backup = unbounded.into_inclusive_with_backup(d("2023-01-01")..=d("2023-01-08")); + assert_eq!(with_backup, d("2023-01-01")..=d("2023-01-08")); + } + + #[test] + fn test_into_inclusive_pair() { + // Basic inclusive range + let inclusive: UniversalRange = (1..=5).into(); + assert_eq!(inclusive.into_inclusive_pair(), (Some(1), Some(5))); + + // Basic exclusive range + let exclusive: UniversalRange = (1..6).into(); + assert_eq!(exclusive.into_inclusive_pair(), (Some(1), Some(5))); + + // Unbounded range + let unbounded = UniversalRange::::full(); + assert_eq!(unbounded.into_inclusive_pair(), (None, None)); + + // Start-only range + let start_only = UniversalRange { + start: Some(1), + end: Bound::Unbounded, + }; + assert_eq!(start_only.into_inclusive_pair(), (Some(1), None)); + + // End-only range (exclusive) + let end_only = UniversalRange { + start: None, + end: Bound::Excluded(5), + }; + assert_eq!(end_only.into_inclusive_pair(), (None, Some(4))); + + // End-only range (inclusive) + let end_only_inclusive = UniversalRange { + start: None, + end: Bound::Included(5), + }; + assert_eq!(end_only_inclusive.into_inclusive_pair(), (None, Some(5))); + } + + #[test] + fn test_exclusive_to_inclusive_conversion() { + // Normal case + assert_eq!(exclusive_range_to_inclusive(1..5), 1..=4); + + // Single-element range + assert_eq!(exclusive_range_to_inclusive(1..2), 1..=1); + + // Empty range at start + let empty_start = exclusive_range_to_inclusive(0..0); + assert!(empty_start.is_empty()); + + // Empty range at end of i32 + let empty_end = exclusive_range_to_inclusive(i32::MAX..i32::MAX); + assert!(empty_end.is_empty()); + + // Range ending at i32::MAX + assert_eq!( + exclusive_range_to_inclusive(0..i32::MAX), + 0..=(i32::MAX - 1) + ); + + // Minimal non-empty range + assert_eq!(exclusive_range_to_inclusive(5..6), 5..=5); + } + + #[test] + fn test_inclusive_to_exclusive_conversion() { + // Normal case + assert_eq!(inclusive_range_to_exclusive(1..=4), 1..5); + + // Single element range + assert_eq!(inclusive_range_to_exclusive(1..=1), 1..2); + + // can't represent (i32::MAX..=i32::MAX) as an exclusive range + // so no test for that + + // Range ending at i32::MAX + assert_eq!( + inclusive_range_to_exclusive(0..=i32::MAX), + 0..i32::MAX // Note: this saturates at MAX instead of overflowing + ); + + // Range ending one before MAX + assert_eq!( + inclusive_range_to_exclusive(0..=(i32::MAX - 1)), + 0..i32::MAX + ); + + // Test with minimum values + assert_eq!( + inclusive_range_to_exclusive(i32::MIN..=i32::MIN), + i32::MIN..(i32::MIN + 1) + ); + } +} diff --git a/stats/stats/src/tests/init_db.rs b/stats/stats/src/tests/init_db.rs index 655511e45..29473a678 100644 --- a/stats/stats/src/tests/init_db.rs +++ b/stats/stats/src/tests/init_db.rs @@ -1,5 +1,7 @@ use blockscout_service_launcher::test_database::TestDbGuard; +use crate::utils::MarkedDbConnection; + pub async fn init_db_all(name: &str) -> (TestDbGuard, TestDbGuard) { let db = init_db(name).await; let blockscout = @@ -11,3 +13,11 @@ pub async fn init_db_all(name: &str) -> (TestDbGuard, TestDbGuard) { pub async fn init_db(name: &str) -> TestDbGuard { TestDbGuard::new::(name).await } + +pub async fn init_marked_db_all(name: &str) -> (MarkedDbConnection, MarkedDbConnection) { + let (db, blockscout) = init_db_all(name).await; + ( + MarkedDbConnection::from_test_db(&db).unwrap(), + MarkedDbConnection::from_test_db(&blockscout).unwrap(), + ) +} diff --git a/stats/stats/src/tests/simple_test.rs b/stats/stats/src/tests/simple_test.rs index cdd4a0fd0..0f7af1ece 100644 --- a/stats/stats/src/tests/simple_test.rs +++ b/stats/stats/src/tests/simple_test.rs @@ -1,17 +1,22 @@ -use super::{init_db::init_db_all, mock_blockscout::fill_mock_blockscout_data}; +use super::{ + init_db::{init_db_all, init_marked_db_all}, + mock_blockscout::fill_mock_blockscout_data, +}; use crate::{ data_source::{ source::DataSource, types::{BlockscoutMigrations, UpdateContext, UpdateParameters}, }, - get_line_chart_data, get_raw_counters, - types::Timespan, - ChartProperties, MissingDatePolicy, + query_dispatch::QuerySerialized, + range::UniversalRange, + types::{timespans::DateValue, Timespan}, + utils::MarkedDbConnection, + ChartProperties, }; use blockscout_service_launcher::test_database::TestDbGuard; use chrono::{DateTime, NaiveDateTime, Utc}; use pretty_assertions::assert_eq; -use sea_orm::DatabaseConnection; +use stats_proto::blockscout::stats::v1::Point; use std::{fmt::Debug, str::FromStr}; pub fn map_str_tuple_to_owned(l: Vec<(&str, &str)>) -> Vec<(String, String)> { @@ -33,7 +38,7 @@ pub async fn simple_test_chart( expected: Vec<(&str, &str)>, ) -> (TestDbGuard, TestDbGuard) where - C: DataSource + ChartProperties, + C: DataSource + ChartProperties + QuerySerialized>, C::Resolution: Ord + Clone + Debug, { simple_test_chart_inner::(test_name, expected, BlockscoutMigrations::latest()).await @@ -50,7 +55,7 @@ pub async fn simple_test_chart_with_migration_variants( test_name_base: &str, expected: Vec<(&str, &str)>, ) where - C: DataSource + ChartProperties, + C: DataSource + ChartProperties + QuerySerialized>, C::Resolution: Ord + Clone + Debug, { for (i, migrations) in MIGRATIONS_VARIANTS.into_iter().enumerate() { @@ -59,24 +64,27 @@ pub async fn simple_test_chart_with_migration_variants( } } +fn chart_output_to_expected(output: Vec) -> Vec<(String, String)> { + output.into_iter().map(|p| (p.date, p.value)).collect() +} + async fn simple_test_chart_inner( test_name: &str, expected: Vec<(&str, &str)>, migrations: BlockscoutMigrations, ) -> (TestDbGuard, TestDbGuard) where - C: DataSource + ChartProperties, + C: DataSource + ChartProperties + QuerySerialized>, C::Resolution: Ord + Clone + Debug, { let (current_time, db, blockscout) = prepare_chart_test::(test_name, None).await; let expected = map_str_tuple_to_owned(expected); - let approximate_trailing_points = C::approximate_trailing_points(); let current_date = current_time.date_naive(); fill_mock_blockscout_data(&blockscout, current_date).await; let mut parameters = UpdateParameters { - db: &db, - blockscout: &blockscout, + db: &MarkedDbConnection::from_test_db(&db).unwrap(), + blockscout: &MarkedDbConnection::from_test_db(&blockscout).unwrap(), blockscout_applied_migrations: migrations, update_time_override: Some(current_time), force_full: true, @@ -84,15 +92,11 @@ where let cx = UpdateContext::from_params_now_or_override(parameters.clone()); C::update_recursively(&cx).await.unwrap(); assert_eq!( - &get_chart::( - &db, - None, - None, - C::missing_date_policy(), - false, - approximate_trailing_points, - ) - .await, + &chart_output_to_expected( + C::query_data_static(&cx, UniversalRange::full(), None, false) + .await + .unwrap() + ), &expected ); @@ -100,15 +104,11 @@ where let cx = UpdateContext::from_params_now_or_override(parameters); C::update_recursively(&cx).await.unwrap(); assert_eq!( - &get_chart::( - &db, - None, - None, - C::missing_date_policy(), - false, - approximate_trailing_points, - ) - .await, + &chart_output_to_expected( + C::query_data_static(&cx, UniversalRange::full(), None, false) + .await + .unwrap() + ), &expected ); (db, blockscout) @@ -118,12 +118,12 @@ where /// /// Tests that force update with existing data works correctly pub async fn dirty_force_update_and_check( - db: &DatabaseConnection, - blockscout: &DatabaseConnection, + db: &TestDbGuard, + blockscout: &TestDbGuard, expected: Vec<(&str, &str)>, update_time_override: Option>, ) where - C: DataSource + ChartProperties, + C: DataSource + ChartProperties + QuerySerialized>, C::Resolution: Ord + Clone + Debug, { let _ = tracing_subscriber::fmt::try_init(); @@ -131,11 +131,10 @@ pub async fn dirty_force_update_and_check( // some later time so that the update is not skipped let current_time = update_time_override.unwrap_or(DateTime::from_str("2023-03-01T12:00:01Z").unwrap()); - let approximate_trailing_points = C::approximate_trailing_points(); let parameters = UpdateParameters { - db, - blockscout, + db: &MarkedDbConnection::from_test_db(db).unwrap(), + blockscout: &MarkedDbConnection::from_test_db(blockscout).unwrap(), blockscout_applied_migrations: BlockscoutMigrations::latest(), update_time_override: Some(current_time), force_full: true, @@ -143,15 +142,11 @@ pub async fn dirty_force_update_and_check( let cx = UpdateContext::from_params_now_or_override(parameters.clone()); C::update_recursively(&cx).await.unwrap(); assert_eq!( - &get_chart::( - db, - None, - None, - C::missing_date_policy(), - false, - approximate_trailing_points, - ) - .await, + &chart_output_to_expected( + C::query_data_static(&cx, UniversalRange::full(), None, false) + .await + .unwrap() + ), &expected ); } @@ -170,7 +165,7 @@ pub async fn ranged_test_chart( to: C::Resolution, update_time: Option, ) where - C: DataSource + ChartProperties, + C: DataSource + ChartProperties + QuerySerialized>, C::Resolution: Ord + Clone + Debug, { ranged_test_chart_inner::( @@ -198,7 +193,7 @@ pub async fn ranged_test_chart_with_migration_variants( to: C::Resolution, update_time: Option, ) where - C: DataSource + ChartProperties, + C: DataSource + ChartProperties + QuerySerialized>, C::Resolution: Ord + Clone + Debug, { for (i, migrations) in MIGRATIONS_VARIANTS.into_iter().enumerate() { @@ -223,19 +218,20 @@ async fn ranged_test_chart_inner( update_time: Option, migrations: BlockscoutMigrations, ) where - C: DataSource + ChartProperties, + C: DataSource + ChartProperties + QuerySerialized>, C::Resolution: Ord + Clone + Debug, { let _ = tracing_subscriber::fmt::try_init(); let expected = map_str_tuple_to_owned(expected); - let (db, blockscout) = init_db_all(test_name).await; + let (db, blockscout) = init_marked_db_all(test_name).await; let max_time = DateTime::::from_str("2023-03-01T12:00:00Z").unwrap(); let current_time = update_time.map(|t| t.and_utc()).unwrap_or(max_time); let max_date = max_time.date_naive(); - C::init_recursively(&db, ¤t_time).await.unwrap(); - fill_mock_blockscout_data(&blockscout, max_date).await; - let policy = C::missing_date_policy(); - let approximate_trailing_points = C::approximate_trailing_points(); + let range = { from.into_time_range().start..to.into_time_range().end }; + C::init_recursively(db.connection.as_ref(), ¤t_time) + .await + .unwrap(); + fill_mock_blockscout_data(blockscout.connection.as_ref(), max_date).await; let mut parameters = UpdateParameters { db: &db, @@ -247,15 +243,11 @@ async fn ranged_test_chart_inner( let cx = UpdateContext::from_params_now_or_override(parameters.clone()); C::update_recursively(&cx).await.unwrap(); assert_eq!( - &get_chart::( - &db, - Some(from.clone()), - Some(to.clone()), - policy, - false, - approximate_trailing_points, - ) - .await, + &chart_output_to_expected( + C::query_data_static(&cx, range.clone().into(), None, false) + .await + .unwrap() + ), &expected ); @@ -263,54 +255,23 @@ async fn ranged_test_chart_inner( let cx = UpdateContext::from_params_now_or_override(parameters); C::update_recursively(&cx).await.unwrap(); assert_eq!( - &get_chart::( - &db, - Some(from), - Some(to), - policy, - false, - approximate_trailing_points, - ) - .await, + &chart_output_to_expected( + C::query_data_static(&cx, range.into(), None, false) + .await + .unwrap() + ), &expected ); } -async fn get_chart( - db: &DatabaseConnection, - from: Option, - to: Option, - policy: MissingDatePolicy, - fill_missing_dates: bool, - approximate_trailing_points: u64, -) -> Vec<(String, String)> -where - C: DataSource + ChartProperties, - C::Resolution: Ord + Clone + Debug, -{ - let data = get_line_chart_data::( - db, - &C::name(), - from, - to, - None, - policy, - fill_missing_dates, - approximate_trailing_points, - ) - .await - .unwrap(); - data.into_iter() - .map(|p| (p.timespan.into_date().to_string(), p.value)) - .collect() -} - /// `test_name` must be unique to avoid db clashes -pub async fn simple_test_counter( +pub async fn simple_test_counter( test_name: &str, expected: &str, update_time: Option, -) { +) where + C: DataSource + ChartProperties + QuerySerialized>, +{ simple_test_counter_inner::( test_name, expected, @@ -327,42 +288,46 @@ pub async fn simple_test_counter( /// - db is going to be initialized separately for each variant /// - `_N` will be added to `test_name_base` for each variant /// - the resulting test name must be unique to avoid db clashes -pub async fn simple_test_counter_with_migration_variants( +pub async fn simple_test_counter_with_migration_variants( test_name_base: &str, expected: &str, update_time: Option, -) { +) where + C: DataSource + ChartProperties + QuerySerialized>, +{ for (i, migrations) in MIGRATIONS_VARIANTS.into_iter().enumerate() { let test_name = format!("{test_name_base}_{i}"); simple_test_counter_inner::(&test_name, expected, update_time, migrations).await } } -async fn simple_test_counter_inner( +async fn simple_test_counter_inner( test_name: &str, expected: &str, update_time: Option, migrations: BlockscoutMigrations, -) { +) where + C: DataSource + ChartProperties + QuerySerialized>, +{ let (current_time, db, blockscout) = prepare_chart_test::(test_name, update_time).await; let max_time = DateTime::::from_str("2023-03-01T12:00:00Z").unwrap(); let max_date = max_time.date_naive(); fill_mock_blockscout_data(&blockscout, max_date).await; let mut parameters = UpdateParameters { - db: &db, - blockscout: &blockscout, + db: &MarkedDbConnection::from_test_db(&db).unwrap(), + blockscout: &MarkedDbConnection::from_test_db(&blockscout).unwrap(), blockscout_applied_migrations: migrations, update_time_override: Some(current_time), force_full: true, }; let cx = UpdateContext::from_params_now_or_override(parameters.clone()); C::update_recursively(&cx).await.unwrap(); - assert_eq!(expected, get_counter::(&db).await); + assert_eq!(expected, get_counter::(&cx).await.value); parameters.force_full = false; let cx = UpdateContext::from_params_now_or_override(parameters.clone()); C::update_recursively(&cx).await.unwrap(); - assert_eq!(expected, get_counter::(&db).await); + assert_eq!(expected, get_counter::(&cx).await.value); } pub async fn prepare_chart_test( @@ -378,8 +343,10 @@ pub async fn prepare_chart_test( (init_time, db, blockscout) } -pub async fn get_counter(db: &DatabaseConnection) -> String { - let data = get_raw_counters(db).await.unwrap(); - let data = &data[&C::name()]; - data.value.clone() +pub async fn get_counter>>( + cx: &UpdateContext<'_>, +) -> DateValue { + C::query_data_static(cx, UniversalRange::full(), None, false) + .await + .unwrap() } diff --git a/stats/stats/src/update_group.rs b/stats/stats/src/update_group.rs index 6092cac53..f33d72ba3 100644 --- a/stats/stats/src/update_group.rs +++ b/stats/stats/src/update_group.rs @@ -41,9 +41,9 @@ use thiserror::Error; use tokio::sync::{Mutex, MutexGuard}; use crate::{ - charts::{chart_properties_portrait::imports::ChartKey, ChartPropertiesObject}, + charts::{chart_properties_portrait::imports::ChartKey, ChartObject}, data_source::UpdateParameters, - UpdateError, + ChartError, }; #[derive(Error, Debug, PartialEq)] @@ -68,7 +68,7 @@ pub trait UpdateGroup: core::fmt::Debug { /// Group name (usually equal to type name for simplicity) fn name(&self) -> String; /// List chart properties - members of the group. - fn list_charts(&self) -> Vec; + fn list_charts(&self) -> Vec; /// List mutex ids of group members + their dependencies. /// Dependencies participate in updates, thus access to them needs to be /// synchronized as well. @@ -95,7 +95,7 @@ pub trait UpdateGroup: core::fmt::Debug { &self, params: UpdateParameters<'a>, enabled_charts: &HashSet, - ) -> Result<(), UpdateError>; + ) -> Result<(), ChartError>; } /// Construct update group that implemants [`UpdateGroup`]. The main purpose of the @@ -117,12 +117,12 @@ pub trait UpdateGroup: core::fmt::Debug { /// ```rust /// # use stats::data_source::kinds::{ /// # }; -/// # use stats::{ChartProperties, Named, construct_update_group, types::timespans::DateValue, UpdateError}; +/// # use stats::{ChartProperties, Named, construct_update_group, types::timespans::DateValue, ChartError}; /// # use stats::data_source::{ /// # kinds::{ /// # local_db::{DirectVecLocalDbChartSource, parameters::update::batching::parameters::Batch30Days}, /// # remote_db::{PullAllWithAndSort, RemoteDatabaseSource, StatementFromRange}, -/// # data_manipulation::map::MapToString, +/// # data_manipulation::map::{MapToString, StripExt}, /// # }, /// # types::{UpdateContext, UpdateParameters}, /// # }; @@ -130,7 +130,6 @@ pub trait UpdateGroup: core::fmt::Debug { /// # use chrono::NaiveDate; /// # use entity::sea_orm_active_enums::ChartType; /// # use std::ops::RangeInclusive; -/// # use sea_orm::prelude::DateTimeUtc; /// # use sea_orm::Statement; /// # /// # struct DummyChartProperties; @@ -147,7 +146,7 @@ pub trait UpdateGroup: core::fmt::Debug { /// # } /// # } /// # -/// # type DummyChart = DirectVecLocalDbChartSource; +/// # type DummyChart = DirectVecLocalDbChartSource, Batch30Days, DummyChartProperties>; /// /// construct_update_group!(ExampleUpdateGroup { /// charts: [DummyChart], @@ -197,7 +196,10 @@ pub trait UpdateGroup: core::fmt::Debug { /// ## Example /// /// ```rust -/// # use stats::{ChartProperties, Named, construct_update_group, types::timespans::DateValue, UpdateError}; +/// # use stats::{ +/// # QueryAllBlockTimestampRange, construct_update_group, +/// # types::timespans::DateValue, ChartProperties, Named, ChartError, +/// # }; /// # use stats::data_source::{ /// # kinds::{ /// # local_db::{DirectVecLocalDbChartSource, parameters::update::batching::parameters::Batch30Days}, @@ -205,21 +207,20 @@ pub trait UpdateGroup: core::fmt::Debug { /// # }, /// # types::{UpdateContext, UpdateParameters, BlockscoutMigrations}, /// # }; -/// # use chrono::NaiveDate; +/// # use chrono::{NaiveDate, DateTime, Utc}; /// # use entity::sea_orm_active_enums::ChartType; /// # use std::ops::Range; -/// # use sea_orm::prelude::DateTimeUtc; /// # use sea_orm::Statement; /// /// struct DummyRemoteStatement; /// /// impl StatementFromRange for DummyRemoteStatement { -/// fn get_statement(range: Option>, _: &BlockscoutMigrations) -> Statement { +/// fn get_statement(range: Option>>, _: &BlockscoutMigrations) -> Statement { /// todo!() /// } /// } /// -/// type DummyRemote = RemoteDatabaseSource>; +/// type DummyRemote = RemoteDatabaseSource>; /// /// struct DummyChartProperties; /// @@ -260,10 +261,12 @@ macro_rules! construct_update_group { stringify!($group_name).into() } - fn list_charts(&self) -> ::std::vec::Vec<$crate::ChartPropertiesObject> { + fn list_charts(&self) -> ::std::vec::Vec<$crate::ChartObject> { std::vec![ $( - $crate::ChartPropertiesObject::construct_from_chart::<$member>(), + $crate::ChartObject::construct_from_chart::<$member>( + <$member as $crate::query_dispatch::QuerySerialized>::new_for_dynamic_dispatch() + ), )* ] } @@ -271,15 +274,23 @@ macro_rules! construct_update_group { fn list_dependency_mutex_ids(&self) -> ::std::collections::HashSet { let mut ids = ::std::collections::HashSet::new(); $( - ids.extend(<$member as $crate::data_source::DataSource>::all_dependencies_mutex_ids().into_iter()); + ids.extend( + <$member as $crate::data_source::DataSource>::all_dependencies_mutex_ids() + .into_iter() + ); )* ids } - fn dependency_mutex_ids_of(&self, chart_id: &$crate::ChartKey) -> Option<::std::collections::HashSet> { + fn dependency_mutex_ids_of( + &self, + chart_id: &$crate::ChartKey + ) -> Option<::std::collections::HashSet> { $( if chart_id == &<$member as $crate::ChartProperties>::key() { - return Some(<$member as $crate::data_source::DataSource>::all_dependencies_mutex_ids()); + return Some( + <$member as $crate::data_source::DataSource>::all_dependencies_mutex_ids() + ); } )* return None; @@ -299,7 +310,9 @@ macro_rules! construct_update_group { let current_time = creation_time_override.unwrap_or_else(|| ::chrono::Utc::now()); $( if enabled_charts.contains(&<$member as $crate::ChartProperties>::key()) { - <$member as $crate::data_source::DataSource>::init_recursively(db, ¤t_time).await?; + <$member as $crate::data_source::DataSource>::init_recursively( + db, ¤t_time + ).await?; } )* Ok(()) @@ -307,13 +320,17 @@ macro_rules! construct_update_group { // updates are expected to be unique by group name & update time; this instrumentation // should allow to single out one update process in logs - #[::tracing::instrument(skip_all, fields(update_group=self.name(), update_time), level = tracing::Level::INFO)] + #[::tracing::instrument( + skip_all, + fields(update_group=self.name(), update_time), + level = tracing::Level::INFO + )] async fn update_charts<'a>( &self, params: $crate::data_source::UpdateParameters<'a>, #[allow(unused)] enabled_charts: &::std::collections::HashSet<$crate::ChartKey>, - ) -> Result<(), $crate::UpdateError> { + ) -> Result<(), $crate::ChartError> { let cx = $crate::data_source::UpdateContext::from_params_now_or_override(params); ::tracing::Span::current().record("update_time", ::std::format!("{}",&cx.time)); $( @@ -391,7 +408,7 @@ impl SyncUpdateGroup { } /// See [`UpdateGroup::list_charts``] - pub fn list_charts(&self) -> Vec { + pub fn list_charts(&self) -> Vec { self.inner.list_charts() } @@ -461,7 +478,11 @@ impl SyncUpdateGroup { &self, enabled_charts: &HashSet, ) -> (Vec>, HashSet) { - let members: HashSet = self.list_charts().into_iter().map(|c| c.key).collect(); + let members: HashSet = self + .list_charts() + .into_iter() + .map(|c| c.properties.key) + .collect(); // in-place intersection let enabled_members: HashSet = members .into_iter() @@ -479,12 +500,12 @@ impl SyncUpdateGroup { db: &DatabaseConnection, creation_time_override: Option>, enabled_charts: &HashSet, - ) -> Result<(), UpdateError> { + ) -> Result<(), ChartError> { let (_joint_guard, enabled_members) = self.lock_enabled_dependencies(enabled_charts).await; self.inner .create_charts(db, creation_time_override, &enabled_members) .await - .map_err(UpdateError::StatsDB) + .map_err(ChartError::StatsDB) } /// Ignores unknown names @@ -492,7 +513,7 @@ impl SyncUpdateGroup { &self, params: UpdateParameters<'a>, enabled_charts: &HashSet, - ) -> Result<(), UpdateError> { + ) -> Result<(), ChartError> { let (_joint_guard, enabled_members) = self.lock_enabled_dependencies(enabled_charts).await; tracing::info!( update_group = self.name(), diff --git a/stats/stats/src/utils.rs b/stats/stats/src/utils.rs index 185da50e4..203bb61ff 100644 --- a/stats/stats/src/utils.rs +++ b/stats/stats/src/utils.rs @@ -1,26 +1,54 @@ //! Common utilities used across statistics -use std::ops::{Range, RangeInclusive}; - -use chrono::{NaiveDate, NaiveTime}; -use sea_orm::{prelude::DateTimeUtc, Value}; +use chrono::{DateTime, NaiveDate, NaiveTime, Utc}; +use sea_orm::Value; +use std::{ops::Range, sync::Arc}; // this const is not public in `chrono` for some reason pub const NANOS_PER_SEC: i32 = 1_000_000_000; -pub fn day_start(date: &NaiveDate) -> DateTimeUtc { +pub fn day_start(date: &NaiveDate) -> DateTime { date.and_time(NaiveTime::from_hms_opt(0, 0, 0).expect("correct time")) .and_utc() } -pub fn exclusive_datetime_range_to_inclusive(r: Range) -> RangeInclusive { - // subtract the smallest unit of time to get semantically the same range - // but inclusive - let new_end = r - .end - .checked_sub_signed(chrono::Duration::nanoseconds(1)) - .unwrap_or(DateTimeUtc::MIN_UTC); // saturating sub - r.start..=new_end +// todo: remove marked part if not used until May 2025. +// probably rename to some wrapper of db connection to add some other +// stuff if necessary (or use UpdateContext as a place to extend the context) +/// Database connection with a mark of what database it is. +/// +/// Used to separate caching for different databases to +/// prevent data clashes when running unit tests concurrently. +#[derive(Debug, Clone)] +pub struct MarkedDbConnection { + pub connection: Arc, + pub db_name: String, +} + +impl MarkedDbConnection { + #[cfg(any(feature = "test-utils", test))] + pub fn from_test_db( + guard: &blockscout_service_launcher::test_database::TestDbGuard, + ) -> Option { + Some(Self { + connection: guard.client(), + db_name: guard.db_url().split("/").last()?.to_owned(), + }) + } + + pub fn main_connection(inner: Arc) -> Self { + Self { + connection: inner, + db_name: "main".to_owned(), + } + } + + pub fn in_memory(inner: Arc) -> Self { + Self { + connection: inner, + db_name: "in_memory".to_owned(), + } + } } /// Used inside [`sql_with_range_filter_opt`] @@ -32,7 +60,7 @@ pub fn exclusive_datetime_range_to_inclusive(r: Range) -> RangeIncl /// Vec should be appended to the args. /// String should be inserted in places for filter. pub(crate) fn produce_filter_and_values( - range: Option>, + range: Option>>, filter_by: &str, filter_arg_number_start: usize, ) -> (String, Vec) { @@ -104,8 +132,8 @@ mod test { ("".to_string(), vec![]) ); - let time_1 = DateTimeUtc::from_timestamp(1234567, 0).unwrap(); - let time_2 = DateTimeUtc::from_timestamp(7654321, 0).unwrap(); + let time_1 = DateTime::::from_timestamp(1234567, 0).unwrap(); + let time_2 = DateTime::::from_timestamp(7654321, 0).unwrap(); assert_eq!( produce_filter_and_values(Some(time_1..time_2), "aboba", 123), ( @@ -120,7 +148,7 @@ mod test { const ETH: i64 = 1_000_000_000_000_000_000; - fn naive_sql_selector(range: Option>) -> Statement { + fn naive_sql_selector(range: Option>>) -> Statement { match range { Some(range) => Statement::from_sql_and_values( DbBackend::Postgres, @@ -185,8 +213,8 @@ mod test { #[test] fn sql_with_range_filter_some_works() { let range = Some( - DateTimeUtc::from_timestamp(1234567, 0).unwrap() - ..DateTimeUtc::from_timestamp(7654321, 0).unwrap(), + DateTime::::from_timestamp(1234567, 0).unwrap() + ..DateTime::::from_timestamp(7654321, 0).unwrap(), ); assert_eq!( compact_sql(naive_sql_selector(range.clone())),