From 93c11bc6150e87d7107ca26eef3bf88ed9499689 Mon Sep 17 00:00:00 2001 From: Rabbit Date: Mon, 16 Dec 2024 17:15:44 +0800 Subject: [PATCH] chore: update rgbpp hourly statistic --- .../v2/rgbpp_hourly_statistics_controller.rb | 3 +- app/models/cell_dependency.rb | 6 ++- app/models/rgbpp_hourly_statistic.rb | 3 +- .../generate_rgbpp_hourly_statistic_worker.rb | 10 ++-- .../generate_udt_hourly_statistic_worker.rb | 50 +++++++++++-------- ...13053309_create_rgbpp_hourly_statistics.rb | 3 +- db/structure.sql | 3 +- 7 files changed, 45 insertions(+), 33 deletions(-) diff --git a/app/controllers/api/v2/rgbpp_hourly_statistics_controller.rb b/app/controllers/api/v2/rgbpp_hourly_statistics_controller.rb index bfb988b4b..f10ac2cf5 100644 --- a/app/controllers/api/v2/rgbpp_hourly_statistics_controller.rb +++ b/app/controllers/api/v2/rgbpp_hourly_statistics_controller.rb @@ -9,7 +9,8 @@ def index render json: { data: rgbpp_statistics.map do |statistic| { - total_count: statistic.total_count.to_s, + xudt_count: statistic.xudt_count.to_s, + dob_count: statistic.dob_count.to_s, created_at_unixtimestamp: statistic.created_at_unixtimestamp.to_s, } end, diff --git a/app/models/cell_dependency.rb b/app/models/cell_dependency.rb index 0729d5086..845adc6d1 100644 --- a/app/models/cell_dependency.rb +++ b/app/models/cell_dependency.rb @@ -26,12 +26,12 @@ def to_raw # Table name: cell_dependencies # # id :bigint not null, primary key +# contract_id :bigint # ckb_transaction_id :bigint not null # dep_type :integer # contract_cell_id :bigint not null # script_id :bigint -# contract_id :bigint -# implicit :boolean +# implicit :boolean default(TRUE), not null # block_number :bigint # tx_index :integer # contract_analyzed :boolean default(FALSE) @@ -40,6 +40,8 @@ def to_raw # # index_cell_dependencies_on_block_number_and_tx_index (block_number,tx_index) # index_cell_dependencies_on_contract_analyzed (contract_analyzed) +# index_cell_dependencies_on_contract_id (contract_id) +# index_cell_dependencies_on_script_id (script_id) # index_cell_dependencies_on_tx_id_and_cell_id_and_dep_type (ckb_transaction_id,contract_cell_id,dep_type) UNIQUE # index_on_cell_dependencies_contract_cell_block_tx (contract_cell_id,block_number DESC,tx_index DESC) # diff --git a/app/models/rgbpp_hourly_statistic.rb b/app/models/rgbpp_hourly_statistic.rb index afc4fe44c..14c6f0681 100644 --- a/app/models/rgbpp_hourly_statistic.rb +++ b/app/models/rgbpp_hourly_statistic.rb @@ -6,7 +6,8 @@ class RgbppHourlyStatistic < ApplicationRecord # Table name: rgbpp_hourly_statistics # # id :bigint not null, primary key -# total_count :integer default(0) +# xudt_count :integer default(0) +# dob_count :integer default(0) # created_at_unixtimestamp :integer # created_at :datetime not null # updated_at :datetime not null diff --git a/app/workers/generate_rgbpp_hourly_statistic_worker.rb b/app/workers/generate_rgbpp_hourly_statistic_worker.rb index cf0e7f1d8..b61a25c47 100644 --- a/app/workers/generate_rgbpp_hourly_statistic_worker.rb +++ b/app/workers/generate_rgbpp_hourly_statistic_worker.rb @@ -2,13 +2,11 @@ class GenerateRgbppHourlyStatisticWorker include Sidekiq::Job def perform - xudts_count = Udt.published_xudt.joins(:xudt_tag).where("xudt_tags.tags && ARRAY[?]::varchar[]", ["rgb++"]).count - nft_collections_count = TokenCollection.where("tags && ARRAY[?]::varchar[]", ["rgb++"]).count + xudt_count = Udt.published_xudt.joins(:xudt_tag).where("xudt_tags.tags && ARRAY[?]::varchar[]", ["rgb++"]).count + dob_count = TokenCollection.where("tags && ARRAY[?]::varchar[]", ["rgb++"]).count + created_at_unixtimestamp = to_be_counted_date.beginning_of_day.to_i RgbppHourlyStatistic.upsert( - { - total_count: xudts_count + nft_collections_count, - created_at_unixtimestamp: to_be_counted_date.beginning_of_day.to_i, - }, + { xudt_count:, dob_count:, created_at_unixtimestamp: }, unique_by: :created_at_unixtimestamp, ) rescue StandardError => e diff --git a/app/workers/generate_udt_hourly_statistic_worker.rb b/app/workers/generate_udt_hourly_statistic_worker.rb index 5dd7c79f6..3f8f57bbb 100644 --- a/app/workers/generate_udt_hourly_statistic_worker.rb +++ b/app/workers/generate_udt_hourly_statistic_worker.rb @@ -1,23 +1,10 @@ class GenerateUdtHourlyStatisticWorker include Sidekiq::Job - def perform - udt_types = %i[xudt xudt_compatible spore_cell did_cell] - created_at_unixtimestamp = to_be_counted_date.beginning_of_day.to_i + def perform(datetime = nil) ActiveRecord::Base.connection.execute("SET statement_timeout = 0") - Udt.where(udt_type: udt_types, published: true).find_each do |udt| - puts "Generating statistics for #{udt.id}" - UdtHourlyStatistic.upsert( - { - udt_id: udt.id, - amount: calc_amount(udt), - ckb_transactions_count: calc_ckb_transactions_count(udt), - holders_count: calc_holders_count(udt), - created_at_unixtimestamp:, - }, - unique_by: %i[udt_id created_at_unixtimestamp], - ) - end + start_time = to_be_counted_date(datetime) + generate_statistics(start_time) ActiveRecord::Base.connection.execute("RESET statement_timeout") rescue StandardError => e Rails.logger.error "Error occurred during GenerateUdtHourlyStatistic error: #{e.message}" @@ -25,12 +12,31 @@ def perform private - def to_be_counted_date + def to_be_counted_date(datetime) last_record = UdtHourlyStatistic.order(created_at_unixtimestamp: :desc).first if last_record Time.zone.at(last_record.created_at_unixtimestamp) + 1.day else - Time.current.yesterday + datetime.is_a?(String) ? Time.zone.parse(datetime) : Time.current.yesterday + end + end + + def generate_statistics(start_time) + puts "Generating udt hourly statistics for #{start_time}" + statistic_attributes = [] + udt_types = %i[xudt xudt_compatible spore_cell did_cell] + Udt.where(udt_type: udt_types, published: true).find_each do |udt| + statistic_attributes << { + udt_id: udt.id, + amount: calc_amount(udt), + ckb_transactions_count: calc_ckb_transactions_count(udt), + holders_count: calc_holders_count(udt), + created_at_unixtimestamp: start_time.beginning_of_day.to_i, + } + end + + if statistic_attributes.present? + UdtHourlyStatistic.upsert_all(statistic_attributes, unique_by: %i[udt_id created_at_unixtimestamp]) end end @@ -38,9 +44,11 @@ def calc_amount(udt) inputs_amount = 0 outputs_amount = 0 ckb_transaction_ids = udt.ckb_transactions.map(&:id) - ckb_transaction_ids.each_slice(1000) do |ids| - inputs_amount += CellOutput.where(consumed_by_id: ids).sum(:udt_amount) - outputs_amount += CellOutput.where(ckb_transaction_id: ids).sum(:udt_amount) + ckb_transaction_ids.each_slice(5000) do |ids| + batch_inputs_amount = CellOutput.select(:udt_amount).where(consumed_by_id: ids).map(&:udt_amount).compact + inputs_amount += batch_inputs_amount.sum + batch_outputs_amount = CellOutput.select(:udt_amount).where(ckb_transaction_id: ids).map(&:udt_amount).compact + outputs_amount += batch_outputs_amount.sum end [inputs_amount, outputs_amount].max end diff --git a/db/migrate/20241213053309_create_rgbpp_hourly_statistics.rb b/db/migrate/20241213053309_create_rgbpp_hourly_statistics.rb index fcb0f734b..1b12c0c42 100644 --- a/db/migrate/20241213053309_create_rgbpp_hourly_statistics.rb +++ b/db/migrate/20241213053309_create_rgbpp_hourly_statistics.rb @@ -1,7 +1,8 @@ class CreateRgbppHourlyStatistics < ActiveRecord::Migration[7.0] def change create_table :rgbpp_hourly_statistics do |t| - t.integer :total_count, default: 0 + t.integer :xudt_count, default: 0 + t.integer :dob_count, default: 0 t.integer :created_at_unixtimestamp t.timestamps end diff --git a/db/structure.sql b/db/structure.sql index cc1a214c3..fa548b2fd 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -2350,7 +2350,8 @@ ALTER SEQUENCE public.reject_reasons_id_seq OWNED BY public.reject_reasons.id; CREATE TABLE public.rgbpp_hourly_statistics ( id bigint NOT NULL, - total_count integer DEFAULT 0, + xudt_count integer DEFAULT 0, + dob_count integer DEFAULT 0, created_at_unixtimestamp integer, created_at timestamp(6) without time zone NOT NULL, updated_at timestamp(6) without time zone NOT NULL