From e6de4185acd5752b2096ac29091591a4f41f05c7 Mon Sep 17 00:00:00 2001 From: Miles Zhang Date: Fri, 23 Aug 2024 10:44:38 +0800 Subject: [PATCH] Issue 779 (#2141) * feat: add tx_index to ckb_transactions Signed-off-by: Miles Zhang * feat: use task to fill tx_index Signed-off-by: Miles Zhang * feat: save tx_index when sync node Signed-off-by: Miles Zhang --------- Signed-off-by: Miles Zhang --- .../ckb_sync/new_node_data_processor.rb | 6 +-- app/models/ckb_transaction.rb | 1 + ...2024448_add_tx_index_to_ckb_transaction.rb | 5 +++ db/structure.sql | 27 ++++++------ lib/tasks/migration/fill_tx_index.rake | 43 +++++++++++++++++++ 5 files changed, 65 insertions(+), 17 deletions(-) create mode 100644 db/migrate/20240822024448_add_tx_index_to_ckb_transaction.rb create mode 100644 lib/tasks/migration/fill_tx_index.rake diff --git a/app/models/ckb_sync/new_node_data_processor.rb b/app/models/ckb_sync/new_node_data_processor.rb index 02597b151..9334ac121 100644 --- a/app/models/ckb_sync/new_node_data_processor.rb +++ b/app/models/ckb_sync/new_node_data_processor.rb @@ -1246,11 +1246,10 @@ def cell_input_attributes(input, ckb_transaction_id, local_block_id, def build_ckb_transactions!(node_block, local_block, inputs, outputs, outputs_data) cycles = CkbSync::Api.instance.get_block_cycles node_block.header.hash ckb_transactions_attributes = [] - tx_index = 0 hashes = [] header_deps = {} witnesses = {} - node_block.transactions.each do |tx| + node_block.transactions.each_with_index do |tx, tx_index| attrs = ckb_transaction_attributes(local_block, tx, tx_index) if cycles attrs[:cycles] = tx_index > 0 ? cycles[tx_index - 1]&.hex : nil @@ -1263,8 +1262,6 @@ def build_ckb_transactions!(node_block, local_block, inputs, outputs, outputs_da inputs[tx_index] = tx.inputs outputs[tx_index] = tx.outputs outputs_data[tx_index] = tx.outputs_data - - tx_index += 1 end # First update status thus we can use upsert later. otherwise, we may not be able to # locate correct record according to tx_hash @@ -1358,6 +1355,7 @@ def ckb_transaction_attributes(local_block, tx, tx_index) is_cellbase: tx_index.zero?, live_cell_changes: live_cell_changes(tx, tx_index), bytes: tx.serialized_size_in_block, + tx_index: tx_index } end diff --git a/app/models/ckb_transaction.rb b/app/models/ckb_transaction.rb index 5e94c1243..55b84e40d 100644 --- a/app/models/ckb_transaction.rb +++ b/app/models/ckb_transaction.rb @@ -275,6 +275,7 @@ def log_deletion_chain # bytes :bigint default(0) # cycles :bigint # confirmation_time :integer +# tx_index :integer # # Indexes # diff --git a/db/migrate/20240822024448_add_tx_index_to_ckb_transaction.rb b/db/migrate/20240822024448_add_tx_index_to_ckb_transaction.rb new file mode 100644 index 000000000..74f7fa8df --- /dev/null +++ b/db/migrate/20240822024448_add_tx_index_to_ckb_transaction.rb @@ -0,0 +1,5 @@ +class AddTxIndexToCkbTransaction < ActiveRecord::Migration[7.0] + def change + add_column :ckb_transactions, :tx_index, :integer + end +end diff --git a/db/structure.sql b/db/structure.sql index c099ed0ba..d28bfb536 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -9,13 +9,6 @@ SET xmloption = content; SET client_min_messages = warning; SET row_security = off; --- --- Name: public; Type: SCHEMA; Schema: -; Owner: - --- - --- *not* creating schema, since initdb creates it - - -- -- Name: btree_gin; Type: EXTENSION; Schema: -; Owner: - -- @@ -1350,7 +1343,8 @@ CREATE TABLE public.ckb_transactions ( tags character varying[] DEFAULT '{}'::character varying[], bytes bigint DEFAULT 0, cycles bigint, - confirmation_time integer + confirmation_time integer, + tx_index integer ) PARTITION BY LIST (tx_status); @@ -1395,7 +1389,8 @@ CREATE TABLE public.ckb_transactions_committed ( tags character varying[] DEFAULT '{}'::character varying[], bytes bigint DEFAULT 0, cycles bigint, - confirmation_time integer + confirmation_time integer, + tx_index integer ); @@ -1420,7 +1415,8 @@ CREATE TABLE public.ckb_transactions_pending ( tags character varying[] DEFAULT '{}'::character varying[], bytes bigint DEFAULT 0, cycles bigint, - confirmation_time integer + confirmation_time integer, + tx_index integer ); @@ -1445,7 +1441,8 @@ CREATE TABLE public.ckb_transactions_proposed ( tags character varying[] DEFAULT '{}'::character varying[], bytes bigint DEFAULT 0, cycles bigint, - confirmation_time integer + confirmation_time integer, + tx_index integer ); @@ -1470,7 +1467,8 @@ CREATE TABLE public.ckb_transactions_rejected ( tags character varying[] DEFAULT '{}'::character varying[], bytes bigint DEFAULT 0, cycles bigint, - confirmation_time integer + confirmation_time integer, + tx_index integer ); @@ -6098,4 +6096,7 @@ INSERT INTO "schema_migrations" (version) VALUES ('20240709131020'), ('20240709131132'), ('20240709131713'), -('20240709142013'); +('20240709142013'), +('20240822024448'); + + diff --git a/lib/tasks/migration/fill_tx_index.rake b/lib/tasks/migration/fill_tx_index.rake new file mode 100644 index 000000000..f64393695 --- /dev/null +++ b/lib/tasks/migration/fill_tx_index.rake @@ -0,0 +1,43 @@ +namespace :migration do + desc "Usage: RAILS_ENV=production bundle exec rake migration:fill_tx_index" + task fill_tx_index: :environment do + $retry_ids = Set.new + @api = CKB::API.new(host: ENV["CKB_NODE_URL"], + timeout_config: { + open_timeout: 1, read_timeout: 3, + write_timeout: 1 + }) + first_tx = CkbTransaction.tx_committed.where(tx_index: nil).order("block_number asc").select(:block_number).first + last_tx = CkbTransaction.tx_committed.where(tx_index: nil).order("block_number desc").select(:block_number).first + + (first_tx.block_number..last_tx.block_number).to_a.each_slice(100).to_a.each do |range| + fill_missed_tx_index(range, 0) + end; nil + + puts "retry IDS:" + puts $retry_ids.join(",") + puts "done" + end + + def fill_missed_tx_index(range, retry_count) + request_body = + range.map do |number| + ["get_block_by_number", number] + end + response = @api.batch_request(*request_body) + attrs = [] + response.each do |r| + r[:transactions].each_with_index do |tx, index| + attrs << { tx_hash: tx[:hash], tx_status: "committed", tx_index: index } + end + end; nil + CkbTransaction.upsert_all(attrs, unique_by: %i[tx_status tx_hash]) + rescue StandardError => _e + retry_count += 1 + if retry_count > 2 + $retry_ids << range.first + else + fill_missed_tx_index(range, retry_count) + end + end +end