From 1f93ebeaa047f92ec2d003b443e2f0135e9c59e3 Mon Sep 17 00:00:00 2001 From: Rabbit Date: Fri, 13 Dec 2024 11:47:15 +0800 Subject: [PATCH 1/7] feat: add udt hourly statistic --- .../v2/udt_hourly_statistics_controller.rb | 28 +++++++ app/models/udt_hourly_statistic.rb | 40 ++++++++++ .../generate_udt_hourly_statistic_worker.rb | 54 +++++++++++++ config/routes/v2.rb | 1 + ...1212022531_create_udt_hourly_statistics.rb | 14 ++++ db/structure.sql | 79 ++++++++++++++++++- lib/scheduler.rb | 4 + 7 files changed, 217 insertions(+), 3 deletions(-) create mode 100644 app/controllers/api/v2/udt_hourly_statistics_controller.rb create mode 100644 app/models/udt_hourly_statistic.rb create mode 100644 app/workers/generate_udt_hourly_statistic_worker.rb create mode 100644 db/migrate/20241212022531_create_udt_hourly_statistics.rb diff --git a/app/controllers/api/v2/udt_hourly_statistics_controller.rb b/app/controllers/api/v2/udt_hourly_statistics_controller.rb new file mode 100644 index 000000000..6db2a6dca --- /dev/null +++ b/app/controllers/api/v2/udt_hourly_statistics_controller.rb @@ -0,0 +1,28 @@ +module Api + module V2 + class UdtHourlyStatisticsController < BaseController + def show + expires_in 15.minutes, public: true, stale_while_revalidate: 5.minutes, stale_if_error: 5.minutes + + udt = Udt.find_by!(type_hash: params[:id], published: true) + hourly_statistics = + if udt.present? + UdtHourlyStatistic.where(udt:).order(created_at_unixtimestamp: :asc) + else + UdtHourlyStatistic.none + end + + render json: { + data: hourly_statistics.map do |statistic| + { + ckb_transactions_count: statistic.ckb_transactions_count, + amount: statistic.amount, + holders_count: statistic.holders_count, + created_at_unixtimestamp: statistic.created_at_unixtimestamp, + } + end, + } + end + end + end +end diff --git a/app/models/udt_hourly_statistic.rb b/app/models/udt_hourly_statistic.rb new file mode 100644 index 000000000..233f2bb22 --- /dev/null +++ b/app/models/udt_hourly_statistic.rb @@ -0,0 +1,40 @@ +class UdtHourlyStatistic < ApplicationRecord + belongs_to :udt + + def percentage_change(attribute) + yesterday = previous_stat(udt_id, 1) + day_before_yesterday = previous_stat(udt_id, 2) + + return nil unless yesterday && day_before_yesterday + + yesterday_value = yesterday.public_send(attribute) + day_before_yesterday_value = day_before_yesterday.public_send(attribute) + + return nil if day_before_yesterday_value.zero? + + ((yesterday_value - day_before_yesterday_value).to_f / day_before_yesterday_value * 100).round(2) + end + + def previous_stat(udt_id, days_ago) + timestamp = (Time.current - days_ago.days).beginning_of_day.to_i + self.class.find_by(udt_id:, created_at_unixtimestamp: timestamp) + end +end + +# == Schema Information +# +# Table name: udt_hourly_statistics +# +# id :bigint not null, primary key +# udt_id :bigint not null +# ckb_transactions_count :integer default(0) +# amount :decimal(40, ) default(0) +# holders_count :integer default(0) +# created_at_unixtimestamp :integer +# created_at :datetime not null +# updated_at :datetime not null +# +# Indexes +# +# index_on_udt_id_and_unixtimestamp (udt_id,created_at_unixtimestamp) UNIQUE +# diff --git a/app/workers/generate_udt_hourly_statistic_worker.rb b/app/workers/generate_udt_hourly_statistic_worker.rb new file mode 100644 index 000000000..82df72a49 --- /dev/null +++ b/app/workers/generate_udt_hourly_statistic_worker.rb @@ -0,0 +1,54 @@ +class GenerateUdtHourlyStatisticWorker + include Sidekiq::Job + + def perform + statistic_attributes = [] + udt_types = %i[xudt xudt_compatible spore_cell did_cell] + Udt.where(udt_type: udt_types, published: true).find_each do |udt| + statistic_attributes << { + udt_id: udt.id, + amount: calc_amount(udt), + ckb_transactions_count: calc_ckb_transactions_count(udt), + holders_count: calc_holders_count(udt), + created_at_unixtimestamp: to_be_counted_date.beginning_of_day.to_i, + } + end + + if statistic_attributes.present? + DailyStatisticGenerator.upsert_all(statistic_attributes, + unique_by: %i[udt_id created_at_unixtimestamp]) + end + rescue StandardError => e + Rails.logger.error "Error occurred during GenerateUdtHourlyStatistic error: #{e.message}" + end + + private + + def to_be_counted_date + last_record = UdtHourlyStatistic.order(created_at_unixtimestamp: :desc).first + if last_record + Time.zone.at(last_record.created_at_unixtimestamp) + 1.day + else + Time.current.yesterday + end + end + + def calc_amount(udt) + inputs_amount = 0 + outputs_amount = 0 + ckb_transaction_ids = udt.ckb_transactions.map(&:id) + ckb_transaction_ids.each_slice(1000) do |ids| + inputs_amount += CellOutput.where(consumed_by_id: ids).sum(:udt_amount) + outputs_amount += CellOutput.where(ckb_transaction_id: ids).sum(:udt_amount) + end + [inputs_amount, outputs_amount].max + end + + def calc_ckb_transactions_count(udt) + udt.ckb_transactions.count + end + + def calc_holders_count(udt) + udt.udt_holder_allocations.sum("ckb_holder_count + btc_holder_count") + end +end diff --git a/config/routes/v2.rb b/config/routes/v2.rb index 6a74404f9..1279db51d 100644 --- a/config/routes/v2.rb +++ b/config/routes/v2.rb @@ -106,5 +106,6 @@ resources :graph_nodes, param: :node_id, only: %i[index show] resources :graph_channels, only: :index end + resources :udt_hourly_statistics, only: :show end end diff --git a/db/migrate/20241212022531_create_udt_hourly_statistics.rb b/db/migrate/20241212022531_create_udt_hourly_statistics.rb new file mode 100644 index 000000000..4cb58413f --- /dev/null +++ b/db/migrate/20241212022531_create_udt_hourly_statistics.rb @@ -0,0 +1,14 @@ +class CreateUdtHourlyStatistics < ActiveRecord::Migration[7.0] + def change + create_table :udt_hourly_statistics do |t| + t.bigint :udt_id, null: false + t.integer :ckb_transactions_count, default: 0 + t.decimal :amount, precision: 40, default: 0.0 + t.integer :holders_count, default: 0 + t.integer :created_at_unixtimestamp + t.timestamps + end + + add_index :udt_hourly_statistics, %i[udt_id created_at_unixtimestamp], name: "index_on_udt_id_and_unixtimestamp", unique: true + end +end diff --git a/db/structure.sql b/db/structure.sql index c1f18c9b6..f67ebfbde 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -1054,12 +1054,12 @@ ALTER SEQUENCE public.cell_data_cell_output_id_seq OWNED BY public.cell_data.cel CREATE TABLE public.cell_dependencies ( id bigint NOT NULL, + contract_id bigint, ckb_transaction_id bigint NOT NULL, dep_type integer, contract_cell_id bigint NOT NULL, script_id bigint, - contract_id bigint, - implicit boolean, + implicit boolean DEFAULT true NOT NULL, block_number bigint, tx_index integer, contract_analyzed boolean DEFAULT false @@ -2698,6 +2698,41 @@ CREATE SEQUENCE public.udt_holder_allocations_id_seq ALTER SEQUENCE public.udt_holder_allocations_id_seq OWNED BY public.udt_holder_allocations.id; +-- +-- Name: udt_hourly_statistics; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.udt_hourly_statistics ( + id bigint NOT NULL, + udt_id bigint NOT NULL, + ckb_transactions_count integer DEFAULT 0, + amount numeric(40,0) DEFAULT 0.0, + holders_count integer DEFAULT 0, + created_at_unixtimestamp integer, + created_at timestamp(6) without time zone NOT NULL, + updated_at timestamp(6) without time zone NOT NULL +); + + +-- +-- Name: udt_hourly_statistics_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.udt_hourly_statistics_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: udt_hourly_statistics_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.udt_hourly_statistics_id_seq OWNED BY public.udt_hourly_statistics.id; + + -- -- Name: udt_transactions; Type: TABLE; Schema: public; Owner: - -- @@ -3364,6 +3399,13 @@ ALTER TABLE ONLY public.udt_accounts ALTER COLUMN id SET DEFAULT nextval('public ALTER TABLE ONLY public.udt_holder_allocations ALTER COLUMN id SET DEFAULT nextval('public.udt_holder_allocations_id_seq'::regclass); +-- +-- Name: udt_hourly_statistics id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.udt_hourly_statistics ALTER COLUMN id SET DEFAULT nextval('public.udt_hourly_statistics_id_seq'::regclass); + + -- -- Name: udt_verifications id; Type: DEFAULT; Schema: public; Owner: - -- @@ -3950,6 +3992,14 @@ ALTER TABLE ONLY public.udt_holder_allocations ADD CONSTRAINT udt_holder_allocations_pkey PRIMARY KEY (id); +-- +-- Name: udt_hourly_statistics udt_hourly_statistics_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.udt_hourly_statistics + ADD CONSTRAINT udt_hourly_statistics_pkey PRIMARY KEY (id); + + -- -- Name: udt_verifications udt_verifications_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -4792,6 +4842,20 @@ CREATE INDEX index_cell_dependencies_on_block_number_and_tx_index ON public.cell CREATE INDEX index_cell_dependencies_on_contract_analyzed ON public.cell_dependencies USING btree (contract_analyzed); +-- +-- Name: index_cell_dependencies_on_contract_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_cell_dependencies_on_contract_id ON public.cell_dependencies USING btree (contract_id); + + +-- +-- Name: index_cell_dependencies_on_script_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_cell_dependencies_on_script_id ON public.cell_dependencies USING btree (script_id); + + -- -- Name: index_cell_dependencies_on_tx_id_and_cell_id_and_dep_type; Type: INDEX; Schema: public; Owner: - -- @@ -5044,6 +5108,13 @@ CREATE UNIQUE INDEX index_omiga_inscription_infos_on_udt_hash ON public.omiga_in CREATE INDEX index_on_cell_dependencies_contract_cell_block_tx ON public.cell_dependencies USING btree (contract_cell_id, block_number DESC, tx_index DESC); +-- +-- Name: index_on_udt_id_and_unixtimestamp; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX index_on_udt_id_and_unixtimestamp ON public.udt_hourly_statistics USING btree (udt_id, created_at_unixtimestamp); + + -- -- Name: index_portfolios_on_user_id_and_address_id; Type: INDEX; Schema: public; Owner: - -- @@ -6127,6 +6198,7 @@ INSERT INTO "schema_migrations" (version) VALUES ('20240918033146'), ('20240920094807'), ('20240924065539'), +('20241009081935'), ('20241012014906'), ('20241023055256'), ('20241023063536'), @@ -6141,6 +6213,7 @@ INSERT INTO "schema_migrations" (version) VALUES ('20241129000339'), ('20241129032447'), ('20241202072604'), -('20241205023729'); +('20241205023729'), +('20241212022531'); diff --git a/lib/scheduler.rb b/lib/scheduler.rb index 9afe16d6f..36f3060eb 100644 --- a/lib/scheduler.rb +++ b/lib/scheduler.rb @@ -136,4 +136,8 @@ def call_worker(clz) call_worker FiberGraphDetectWorker end +s.cron "0 8 * * *" do + call_worker GenerateUdtHourlyStatisticWorker +end + s.join From 69fd5b528cae7a6392ca1cbbeebf61cacb82bdde Mon Sep 17 00:00:00 2001 From: Rabbit Date: Fri, 13 Dec 2024 13:59:05 +0800 Subject: [PATCH 2/7] feat: add rgbpp hourly statistic --- .../v2/rgbpp_hourly_statistics_controller.rb | 20 +++++++ app/models/rgbpp_hourly_statistic.rb | 17 ++++++ .../generate_rgbpp_hourly_statistic_worker.rb | 28 +++++++++ .../generate_udt_hourly_statistic_worker.rb | 9 ++- config/routes/v2.rb | 1 + ...13053309_create_rgbpp_hourly_statistics.rb | 11 ++++ db/structure.sql | 57 ++++++++++++++++++- lib/scheduler.rb | 6 +- 8 files changed, 141 insertions(+), 8 deletions(-) create mode 100644 app/controllers/api/v2/rgbpp_hourly_statistics_controller.rb create mode 100644 app/models/rgbpp_hourly_statistic.rb create mode 100644 app/workers/generate_rgbpp_hourly_statistic_worker.rb create mode 100644 db/migrate/20241213053309_create_rgbpp_hourly_statistics.rb diff --git a/app/controllers/api/v2/rgbpp_hourly_statistics_controller.rb b/app/controllers/api/v2/rgbpp_hourly_statistics_controller.rb new file mode 100644 index 000000000..07e691812 --- /dev/null +++ b/app/controllers/api/v2/rgbpp_hourly_statistics_controller.rb @@ -0,0 +1,20 @@ +module Api + module V2 + class RgbppHourlyStatisticsController < BaseController + def index + expires_in 15.minutes, public: true, stale_while_revalidate: 5.minutes, stale_if_error: 5.minutes + + rgbpp_statistics = RgbppHourlyStatistic.order(created_at_unixtimestamp: :asc) + + render json: { + data: rgbpp_statistics.map do |statistic| + { + total_count: statistic.total_count, + created_at_unixtimestamp: statistic.created_at_unixtimestamp, + } + end, + } + end + end + end +end diff --git a/app/models/rgbpp_hourly_statistic.rb b/app/models/rgbpp_hourly_statistic.rb new file mode 100644 index 000000000..afc4fe44c --- /dev/null +++ b/app/models/rgbpp_hourly_statistic.rb @@ -0,0 +1,17 @@ +class RgbppHourlyStatistic < ApplicationRecord +end + +# == Schema Information +# +# Table name: rgbpp_hourly_statistics +# +# id :bigint not null, primary key +# total_count :integer default(0) +# created_at_unixtimestamp :integer +# created_at :datetime not null +# updated_at :datetime not null +# +# Indexes +# +# index_rgbpp_hourly_statistics_on_created_at_unixtimestamp (created_at_unixtimestamp) UNIQUE +# diff --git a/app/workers/generate_rgbpp_hourly_statistic_worker.rb b/app/workers/generate_rgbpp_hourly_statistic_worker.rb new file mode 100644 index 000000000..cf0e7f1d8 --- /dev/null +++ b/app/workers/generate_rgbpp_hourly_statistic_worker.rb @@ -0,0 +1,28 @@ +class GenerateRgbppHourlyStatisticWorker + include Sidekiq::Job + + def perform + xudts_count = Udt.published_xudt.joins(:xudt_tag).where("xudt_tags.tags && ARRAY[?]::varchar[]", ["rgb++"]).count + nft_collections_count = TokenCollection.where("tags && ARRAY[?]::varchar[]", ["rgb++"]).count + RgbppHourlyStatistic.upsert( + { + total_count: xudts_count + nft_collections_count, + created_at_unixtimestamp: to_be_counted_date.beginning_of_day.to_i, + }, + unique_by: :created_at_unixtimestamp, + ) + rescue StandardError => e + Rails.logger.error "Error occurred during GenerateRgbppHourlyStatistic error: #{e.message}" + end + + private + + def to_be_counted_date + last_record = UdtHourlyStatistic.order(created_at_unixtimestamp: :desc).first + if last_record + Time.zone.at(last_record.created_at_unixtimestamp) + 1.day + else + Time.current.yesterday + end + end +end diff --git a/app/workers/generate_udt_hourly_statistic_worker.rb b/app/workers/generate_udt_hourly_statistic_worker.rb index 82df72a49..3330795bb 100644 --- a/app/workers/generate_udt_hourly_statistic_worker.rb +++ b/app/workers/generate_udt_hourly_statistic_worker.rb @@ -4,19 +4,22 @@ class GenerateUdtHourlyStatisticWorker def perform statistic_attributes = [] udt_types = %i[xudt xudt_compatible spore_cell did_cell] + created_at_unixtimestamp = to_be_counted_date.beginning_of_day.to_i Udt.where(udt_type: udt_types, published: true).find_each do |udt| statistic_attributes << { udt_id: udt.id, amount: calc_amount(udt), ckb_transactions_count: calc_ckb_transactions_count(udt), holders_count: calc_holders_count(udt), - created_at_unixtimestamp: to_be_counted_date.beginning_of_day.to_i, + created_at_unixtimestamp:, } end if statistic_attributes.present? - DailyStatisticGenerator.upsert_all(statistic_attributes, - unique_by: %i[udt_id created_at_unixtimestamp]) + DailyStatisticGenerator.upsert_all( + statistic_attributes, + unique_by: %i[udt_id created_at_unixtimestamp], + ) end rescue StandardError => e Rails.logger.error "Error occurred during GenerateUdtHourlyStatistic error: #{e.message}" diff --git a/config/routes/v2.rb b/config/routes/v2.rb index 1279db51d..247c8d9c6 100644 --- a/config/routes/v2.rb +++ b/config/routes/v2.rb @@ -107,5 +107,6 @@ resources :graph_channels, only: :index end resources :udt_hourly_statistics, only: :show + resources :rgbpp_hourly_statistics, only: :index end end diff --git a/db/migrate/20241213053309_create_rgbpp_hourly_statistics.rb b/db/migrate/20241213053309_create_rgbpp_hourly_statistics.rb new file mode 100644 index 000000000..fcb0f734b --- /dev/null +++ b/db/migrate/20241213053309_create_rgbpp_hourly_statistics.rb @@ -0,0 +1,11 @@ +class CreateRgbppHourlyStatistics < ActiveRecord::Migration[7.0] + def change + create_table :rgbpp_hourly_statistics do |t| + t.integer :total_count, default: 0 + t.integer :created_at_unixtimestamp + t.timestamps + end + + add_index :rgbpp_hourly_statistics, :created_at_unixtimestamp, unique: true + end +end diff --git a/db/structure.sql b/db/structure.sql index f67ebfbde..cc1a214c3 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -2344,6 +2344,38 @@ CREATE SEQUENCE public.reject_reasons_id_seq ALTER SEQUENCE public.reject_reasons_id_seq OWNED BY public.reject_reasons.id; +-- +-- Name: rgbpp_hourly_statistics; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.rgbpp_hourly_statistics ( + id bigint NOT NULL, + total_count integer DEFAULT 0, + created_at_unixtimestamp integer, + created_at timestamp(6) without time zone NOT NULL, + updated_at timestamp(6) without time zone NOT NULL +); + + +-- +-- Name: rgbpp_hourly_statistics_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.rgbpp_hourly_statistics_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: rgbpp_hourly_statistics_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.rgbpp_hourly_statistics_id_seq OWNED BY public.rgbpp_hourly_statistics.id; + + -- -- Name: rolling_avg_block_time; Type: MATERIALIZED VIEW; Schema: public; Owner: - -- @@ -3336,6 +3368,13 @@ ALTER TABLE ONLY public.portfolios ALTER COLUMN id SET DEFAULT nextval('public.p ALTER TABLE ONLY public.reject_reasons ALTER COLUMN id SET DEFAULT nextval('public.reject_reasons_id_seq'::regclass); +-- +-- Name: rgbpp_hourly_statistics id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.rgbpp_hourly_statistics ALTER COLUMN id SET DEFAULT nextval('public.rgbpp_hourly_statistics_id_seq'::regclass); + + -- -- Name: statistic_infos id; Type: DEFAULT; Schema: public; Owner: - -- @@ -3912,6 +3951,14 @@ ALTER TABLE ONLY public.reject_reasons ADD CONSTRAINT reject_reasons_pkey PRIMARY KEY (id); +-- +-- Name: rgbpp_hourly_statistics rgbpp_hourly_statistics_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.rgbpp_hourly_statistics + ADD CONSTRAINT rgbpp_hourly_statistics_pkey PRIMARY KEY (id); + + -- -- Name: schema_migrations schema_migrations_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -5129,6 +5176,13 @@ CREATE UNIQUE INDEX index_portfolios_on_user_id_and_address_id ON public.portfol CREATE UNIQUE INDEX index_reject_reasons_on_ckb_transaction_id ON public.reject_reasons USING btree (ckb_transaction_id); +-- +-- Name: index_rgbpp_hourly_statistics_on_created_at_unixtimestamp; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX index_rgbpp_hourly_statistics_on_created_at_unixtimestamp ON public.rgbpp_hourly_statistics USING btree (created_at_unixtimestamp); + + -- -- Name: index_rolling_avg_block_time_on_timestamp; Type: INDEX; Schema: public; Owner: - -- @@ -6214,6 +6268,7 @@ INSERT INTO "schema_migrations" (version) VALUES ('20241129032447'), ('20241202072604'), ('20241205023729'), -('20241212022531'); +('20241212022531'), +('20241213053309'); diff --git a/lib/scheduler.rb b/lib/scheduler.rb index 36f3060eb..d1dd33d5a 100644 --- a/lib/scheduler.rb +++ b/lib/scheduler.rb @@ -32,6 +32,8 @@ def call_worker(clz) s.cron "0 8 * * *" do call_worker Charts::DailyStatistic + call_worker GenerateUdtHourlyStatisticWorker + call_worker GenerateRgbppHourlyStatisticWorker end s.every "10m", overlap: false do @@ -136,8 +138,4 @@ def call_worker(clz) call_worker FiberGraphDetectWorker end -s.cron "0 8 * * *" do - call_worker GenerateUdtHourlyStatisticWorker -end - s.join From 6733cd67e1c960733e0747a84c472035c1ec2638 Mon Sep 17 00:00:00 2001 From: Rabbit Date: Fri, 13 Dec 2024 16:39:16 +0800 Subject: [PATCH 3/7] fix: add statement_timeout --- .../generate_udt_hourly_statistic_worker.rb | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/app/workers/generate_udt_hourly_statistic_worker.rb b/app/workers/generate_udt_hourly_statistic_worker.rb index 3330795bb..944a2a878 100644 --- a/app/workers/generate_udt_hourly_statistic_worker.rb +++ b/app/workers/generate_udt_hourly_statistic_worker.rb @@ -2,25 +2,23 @@ class GenerateUdtHourlyStatisticWorker include Sidekiq::Job def perform - statistic_attributes = [] udt_types = %i[xudt xudt_compatible spore_cell did_cell] created_at_unixtimestamp = to_be_counted_date.beginning_of_day.to_i + ActiveRecord::Base.connection.execute("SET statement_timeout = 0") Udt.where(udt_type: udt_types, published: true).find_each do |udt| - statistic_attributes << { - udt_id: udt.id, - amount: calc_amount(udt), - ckb_transactions_count: calc_ckb_transactions_count(udt), - holders_count: calc_holders_count(udt), - created_at_unixtimestamp:, - } - end - - if statistic_attributes.present? - DailyStatisticGenerator.upsert_all( - statistic_attributes, + puts "Generating statistics for #{udt.id}" + RgbppHourlyStatistic.upsert( + { + udt_id: udt.id, + amount: calc_amount(udt), + ckb_transactions_count: calc_ckb_transactions_count(udt), + holders_count: calc_holders_count(udt), + created_at_unixtimestamp:, + }, unique_by: %i[udt_id created_at_unixtimestamp], ) end + ActiveRecord::Base.connection.execute("RESET statement_timeout") rescue StandardError => e Rails.logger.error "Error occurred during GenerateUdtHourlyStatistic error: #{e.message}" end From 64a52d334b080b6c3e02ac7c1ccc1095801d8369 Mon Sep 17 00:00:00 2001 From: Rabbit Date: Fri, 13 Dec 2024 16:48:53 +0800 Subject: [PATCH 4/7] fix: udt hourly statistic --- app/workers/generate_udt_hourly_statistic_worker.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/generate_udt_hourly_statistic_worker.rb b/app/workers/generate_udt_hourly_statistic_worker.rb index 944a2a878..5dd7c79f6 100644 --- a/app/workers/generate_udt_hourly_statistic_worker.rb +++ b/app/workers/generate_udt_hourly_statistic_worker.rb @@ -7,7 +7,7 @@ def perform ActiveRecord::Base.connection.execute("SET statement_timeout = 0") Udt.where(udt_type: udt_types, published: true).find_each do |udt| puts "Generating statistics for #{udt.id}" - RgbppHourlyStatistic.upsert( + UdtHourlyStatistic.upsert( { udt_id: udt.id, amount: calc_amount(udt), From b33687143612348fddc2d1531db4871fd1ac355b Mon Sep 17 00:00:00 2001 From: Rabbit Date: Fri, 13 Dec 2024 17:16:37 +0800 Subject: [PATCH 5/7] fix: udt hourly statistic api response --- .../api/v2/rgbpp_hourly_statistics_controller.rb | 4 ++-- .../api/v2/udt_hourly_statistics_controller.rb | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/controllers/api/v2/rgbpp_hourly_statistics_controller.rb b/app/controllers/api/v2/rgbpp_hourly_statistics_controller.rb index 07e691812..bfb988b4b 100644 --- a/app/controllers/api/v2/rgbpp_hourly_statistics_controller.rb +++ b/app/controllers/api/v2/rgbpp_hourly_statistics_controller.rb @@ -9,8 +9,8 @@ def index render json: { data: rgbpp_statistics.map do |statistic| { - total_count: statistic.total_count, - created_at_unixtimestamp: statistic.created_at_unixtimestamp, + total_count: statistic.total_count.to_s, + created_at_unixtimestamp: statistic.created_at_unixtimestamp.to_s, } end, } diff --git a/app/controllers/api/v2/udt_hourly_statistics_controller.rb b/app/controllers/api/v2/udt_hourly_statistics_controller.rb index 6db2a6dca..00ac149aa 100644 --- a/app/controllers/api/v2/udt_hourly_statistics_controller.rb +++ b/app/controllers/api/v2/udt_hourly_statistics_controller.rb @@ -15,10 +15,10 @@ def show render json: { data: hourly_statistics.map do |statistic| { - ckb_transactions_count: statistic.ckb_transactions_count, - amount: statistic.amount, - holders_count: statistic.holders_count, - created_at_unixtimestamp: statistic.created_at_unixtimestamp, + ckb_transactions_count: statistic.ckb_transactions_count.to_s, + amount: statistic.amount.to_s, + holders_count: statistic.holders_count.to_s, + created_at_unixtimestamp: statistic.created_at_unixtimestamp.to_s, } end, } From 93c11bc6150e87d7107ca26eef3bf88ed9499689 Mon Sep 17 00:00:00 2001 From: Rabbit Date: Mon, 16 Dec 2024 17:15:44 +0800 Subject: [PATCH 6/7] chore: update rgbpp hourly statistic --- .../v2/rgbpp_hourly_statistics_controller.rb | 3 +- app/models/cell_dependency.rb | 6 ++- app/models/rgbpp_hourly_statistic.rb | 3 +- .../generate_rgbpp_hourly_statistic_worker.rb | 10 ++-- .../generate_udt_hourly_statistic_worker.rb | 50 +++++++++++-------- ...13053309_create_rgbpp_hourly_statistics.rb | 3 +- db/structure.sql | 3 +- 7 files changed, 45 insertions(+), 33 deletions(-) diff --git a/app/controllers/api/v2/rgbpp_hourly_statistics_controller.rb b/app/controllers/api/v2/rgbpp_hourly_statistics_controller.rb index bfb988b4b..f10ac2cf5 100644 --- a/app/controllers/api/v2/rgbpp_hourly_statistics_controller.rb +++ b/app/controllers/api/v2/rgbpp_hourly_statistics_controller.rb @@ -9,7 +9,8 @@ def index render json: { data: rgbpp_statistics.map do |statistic| { - total_count: statistic.total_count.to_s, + xudt_count: statistic.xudt_count.to_s, + dob_count: statistic.dob_count.to_s, created_at_unixtimestamp: statistic.created_at_unixtimestamp.to_s, } end, diff --git a/app/models/cell_dependency.rb b/app/models/cell_dependency.rb index 0729d5086..845adc6d1 100644 --- a/app/models/cell_dependency.rb +++ b/app/models/cell_dependency.rb @@ -26,12 +26,12 @@ def to_raw # Table name: cell_dependencies # # id :bigint not null, primary key +# contract_id :bigint # ckb_transaction_id :bigint not null # dep_type :integer # contract_cell_id :bigint not null # script_id :bigint -# contract_id :bigint -# implicit :boolean +# implicit :boolean default(TRUE), not null # block_number :bigint # tx_index :integer # contract_analyzed :boolean default(FALSE) @@ -40,6 +40,8 @@ def to_raw # # index_cell_dependencies_on_block_number_and_tx_index (block_number,tx_index) # index_cell_dependencies_on_contract_analyzed (contract_analyzed) +# index_cell_dependencies_on_contract_id (contract_id) +# index_cell_dependencies_on_script_id (script_id) # index_cell_dependencies_on_tx_id_and_cell_id_and_dep_type (ckb_transaction_id,contract_cell_id,dep_type) UNIQUE # index_on_cell_dependencies_contract_cell_block_tx (contract_cell_id,block_number DESC,tx_index DESC) # diff --git a/app/models/rgbpp_hourly_statistic.rb b/app/models/rgbpp_hourly_statistic.rb index afc4fe44c..14c6f0681 100644 --- a/app/models/rgbpp_hourly_statistic.rb +++ b/app/models/rgbpp_hourly_statistic.rb @@ -6,7 +6,8 @@ class RgbppHourlyStatistic < ApplicationRecord # Table name: rgbpp_hourly_statistics # # id :bigint not null, primary key -# total_count :integer default(0) +# xudt_count :integer default(0) +# dob_count :integer default(0) # created_at_unixtimestamp :integer # created_at :datetime not null # updated_at :datetime not null diff --git a/app/workers/generate_rgbpp_hourly_statistic_worker.rb b/app/workers/generate_rgbpp_hourly_statistic_worker.rb index cf0e7f1d8..b61a25c47 100644 --- a/app/workers/generate_rgbpp_hourly_statistic_worker.rb +++ b/app/workers/generate_rgbpp_hourly_statistic_worker.rb @@ -2,13 +2,11 @@ class GenerateRgbppHourlyStatisticWorker include Sidekiq::Job def perform - xudts_count = Udt.published_xudt.joins(:xudt_tag).where("xudt_tags.tags && ARRAY[?]::varchar[]", ["rgb++"]).count - nft_collections_count = TokenCollection.where("tags && ARRAY[?]::varchar[]", ["rgb++"]).count + xudt_count = Udt.published_xudt.joins(:xudt_tag).where("xudt_tags.tags && ARRAY[?]::varchar[]", ["rgb++"]).count + dob_count = TokenCollection.where("tags && ARRAY[?]::varchar[]", ["rgb++"]).count + created_at_unixtimestamp = to_be_counted_date.beginning_of_day.to_i RgbppHourlyStatistic.upsert( - { - total_count: xudts_count + nft_collections_count, - created_at_unixtimestamp: to_be_counted_date.beginning_of_day.to_i, - }, + { xudt_count:, dob_count:, created_at_unixtimestamp: }, unique_by: :created_at_unixtimestamp, ) rescue StandardError => e diff --git a/app/workers/generate_udt_hourly_statistic_worker.rb b/app/workers/generate_udt_hourly_statistic_worker.rb index 5dd7c79f6..3f8f57bbb 100644 --- a/app/workers/generate_udt_hourly_statistic_worker.rb +++ b/app/workers/generate_udt_hourly_statistic_worker.rb @@ -1,23 +1,10 @@ class GenerateUdtHourlyStatisticWorker include Sidekiq::Job - def perform - udt_types = %i[xudt xudt_compatible spore_cell did_cell] - created_at_unixtimestamp = to_be_counted_date.beginning_of_day.to_i + def perform(datetime = nil) ActiveRecord::Base.connection.execute("SET statement_timeout = 0") - Udt.where(udt_type: udt_types, published: true).find_each do |udt| - puts "Generating statistics for #{udt.id}" - UdtHourlyStatistic.upsert( - { - udt_id: udt.id, - amount: calc_amount(udt), - ckb_transactions_count: calc_ckb_transactions_count(udt), - holders_count: calc_holders_count(udt), - created_at_unixtimestamp:, - }, - unique_by: %i[udt_id created_at_unixtimestamp], - ) - end + start_time = to_be_counted_date(datetime) + generate_statistics(start_time) ActiveRecord::Base.connection.execute("RESET statement_timeout") rescue StandardError => e Rails.logger.error "Error occurred during GenerateUdtHourlyStatistic error: #{e.message}" @@ -25,12 +12,31 @@ def perform private - def to_be_counted_date + def to_be_counted_date(datetime) last_record = UdtHourlyStatistic.order(created_at_unixtimestamp: :desc).first if last_record Time.zone.at(last_record.created_at_unixtimestamp) + 1.day else - Time.current.yesterday + datetime.is_a?(String) ? Time.zone.parse(datetime) : Time.current.yesterday + end + end + + def generate_statistics(start_time) + puts "Generating udt hourly statistics for #{start_time}" + statistic_attributes = [] + udt_types = %i[xudt xudt_compatible spore_cell did_cell] + Udt.where(udt_type: udt_types, published: true).find_each do |udt| + statistic_attributes << { + udt_id: udt.id, + amount: calc_amount(udt), + ckb_transactions_count: calc_ckb_transactions_count(udt), + holders_count: calc_holders_count(udt), + created_at_unixtimestamp: start_time.beginning_of_day.to_i, + } + end + + if statistic_attributes.present? + UdtHourlyStatistic.upsert_all(statistic_attributes, unique_by: %i[udt_id created_at_unixtimestamp]) end end @@ -38,9 +44,11 @@ def calc_amount(udt) inputs_amount = 0 outputs_amount = 0 ckb_transaction_ids = udt.ckb_transactions.map(&:id) - ckb_transaction_ids.each_slice(1000) do |ids| - inputs_amount += CellOutput.where(consumed_by_id: ids).sum(:udt_amount) - outputs_amount += CellOutput.where(ckb_transaction_id: ids).sum(:udt_amount) + ckb_transaction_ids.each_slice(5000) do |ids| + batch_inputs_amount = CellOutput.select(:udt_amount).where(consumed_by_id: ids).map(&:udt_amount).compact + inputs_amount += batch_inputs_amount.sum + batch_outputs_amount = CellOutput.select(:udt_amount).where(ckb_transaction_id: ids).map(&:udt_amount).compact + outputs_amount += batch_outputs_amount.sum end [inputs_amount, outputs_amount].max end diff --git a/db/migrate/20241213053309_create_rgbpp_hourly_statistics.rb b/db/migrate/20241213053309_create_rgbpp_hourly_statistics.rb index fcb0f734b..1b12c0c42 100644 --- a/db/migrate/20241213053309_create_rgbpp_hourly_statistics.rb +++ b/db/migrate/20241213053309_create_rgbpp_hourly_statistics.rb @@ -1,7 +1,8 @@ class CreateRgbppHourlyStatistics < ActiveRecord::Migration[7.0] def change create_table :rgbpp_hourly_statistics do |t| - t.integer :total_count, default: 0 + t.integer :xudt_count, default: 0 + t.integer :dob_count, default: 0 t.integer :created_at_unixtimestamp t.timestamps end diff --git a/db/structure.sql b/db/structure.sql index cc1a214c3..fa548b2fd 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -2350,7 +2350,8 @@ ALTER SEQUENCE public.reject_reasons_id_seq OWNED BY public.reject_reasons.id; CREATE TABLE public.rgbpp_hourly_statistics ( id bigint NOT NULL, - total_count integer DEFAULT 0, + xudt_count integer DEFAULT 0, + dob_count integer DEFAULT 0, created_at_unixtimestamp integer, created_at timestamp(6) without time zone NOT NULL, updated_at timestamp(6) without time zone NOT NULL From 941f70f33a5e8029bbc8ab593daa2fe89c2cbc3e Mon Sep 17 00:00:00 2001 From: Rabbit Date: Mon, 16 Dec 2024 17:19:15 +0800 Subject: [PATCH 7/7] chore: structure.sql --- db/structure.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/db/structure.sql b/db/structure.sql index fa548b2fd..a9adb8547 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -6253,7 +6253,6 @@ INSERT INTO "schema_migrations" (version) VALUES ('20240918033146'), ('20240920094807'), ('20240924065539'), -('20241009081935'), ('20241012014906'), ('20241023055256'), ('20241023063536'),