From 8b0879a80dd4113936ef9fcf53397e8971d27f9e Mon Sep 17 00:00:00 2001 From: Rabbit Date: Wed, 18 Sep 2024 17:02:25 +0800 Subject: [PATCH 01/15] feat: synchronize peer and channel data in fiber network --- .../api/v2/fiber/channles_controller.rb | 9 + .../api/v2/fiber/peers_controller.rb | 9 + app/models/fiber_channel.rb | 26 ++ app/models/fiber_peer.rb | 18 ++ app/models/fiber_transaction.rb | 16 + app/models/referring_cell.rb | 1 + app/services/fiber.rb | 43 +++ app/workers/fiber_detect_worker.rb | 6 + config/routes/v2.rb | 4 + .../20240918024407_create_fiber_peers.rb | 13 + .../20240918024415_create_fiber_channels.rb | 19 ++ ...0240918033146_create_fiber_transactions.rb | 11 + db/structure.sql | 276 ++++++++++++++++-- lib/scheduler.rb | 4 + test/factories/fiber_channels.rb | 5 + test/factories/fiber_peers.rb | 5 + test/factories/fiber_transactions.rb | 5 + test/models/fiber_channel_test.rb | 7 + test/models/fiber_peer_test.rb | 7 + test/models/fiber_transaction_test.rb | 7 + 20 files changed, 459 insertions(+), 32 deletions(-) create mode 100644 app/controllers/api/v2/fiber/channles_controller.rb create mode 100644 app/controllers/api/v2/fiber/peers_controller.rb create mode 100644 app/models/fiber_channel.rb create mode 100644 app/models/fiber_peer.rb create mode 100644 app/models/fiber_transaction.rb create mode 100644 app/services/fiber.rb create mode 100644 app/workers/fiber_detect_worker.rb create mode 100644 db/migrate/20240918024407_create_fiber_peers.rb create mode 100644 db/migrate/20240918024415_create_fiber_channels.rb create mode 100644 db/migrate/20240918033146_create_fiber_transactions.rb create mode 100644 test/factories/fiber_channels.rb create mode 100644 test/factories/fiber_peers.rb create mode 100644 test/factories/fiber_transactions.rb create mode 100644 test/models/fiber_channel_test.rb create mode 100644 test/models/fiber_peer_test.rb create mode 100644 test/models/fiber_transaction_test.rb diff --git a/app/controllers/api/v2/fiber/channles_controller.rb b/app/controllers/api/v2/fiber/channles_controller.rb new file mode 100644 index 000000000..ecdee23b3 --- /dev/null +++ b/app/controllers/api/v2/fiber/channles_controller.rb @@ -0,0 +1,9 @@ +module Api + module V2 + module Fiber + class ChannlesController < BaseController + def index; end + end + end + end +end diff --git a/app/controllers/api/v2/fiber/peers_controller.rb b/app/controllers/api/v2/fiber/peers_controller.rb new file mode 100644 index 000000000..498960d78 --- /dev/null +++ b/app/controllers/api/v2/fiber/peers_controller.rb @@ -0,0 +1,9 @@ +module Api + module V2 + module Fiber + class PeersController < BaseController + def index; end + end + end + end +end diff --git a/app/models/fiber_channel.rb b/app/models/fiber_channel.rb new file mode 100644 index 000000000..3d48f5cab --- /dev/null +++ b/app/models/fiber_channel.rb @@ -0,0 +1,26 @@ +class FiberChannel < ApplicationRecord + belongs_to :fiber_channel + has_many :fiber_transactions +end + +# == Schema Information +# +# Table name: fiber_channels +# +# id :bigint not null, primary key +# peer_id :string +# channel_id :string +# state_name :string +# state_flags :string default([]), is an Array +# local_balance :decimal(64, 2) default(0.0) +# sent_tlc_balance :decimal(64, 2) default(0.0) +# remote_balance :decimal(64, 2) default(0.0) +# received_tlc_balance :decimal(64, 2) default(0.0) +# shutdown_at :datetime +# created_at :datetime not null +# updated_at :datetime not null +# +# Indexes +# +# index_fiber_channels_on_peer_id_and_channel_id (peer_id,channel_id) UNIQUE +# diff --git a/app/models/fiber_peer.rb b/app/models/fiber_peer.rb new file mode 100644 index 000000000..4a36bd0db --- /dev/null +++ b/app/models/fiber_peer.rb @@ -0,0 +1,18 @@ +class FiberPeer < ApplicationRecord + has_many :fiber_channels, dependent: :destroy, foreign_key: :peer_id + has_many :fiber_transactions +end + +# == Schema Information +# +# Table name: fiber_peers +# +# id :bigint not null, primary key +# name :string +# peer_id :string +# rpc_listening_addr :string +# first_channel_opened_at :datetime +# last_channel_updated_at :datetime +# created_at :datetime not null +# updated_at :datetime not null +# diff --git a/app/models/fiber_transaction.rb b/app/models/fiber_transaction.rb new file mode 100644 index 000000000..4660675df --- /dev/null +++ b/app/models/fiber_transaction.rb @@ -0,0 +1,16 @@ +class FiberTransaction < ApplicationRecord + belongs_to :fiber_channel + belongs_to :fiber_peer +end + +# == Schema Information +# +# Table name: fiber_transactions +# +# id :bigint not null, primary key +# fiber_peer_id :integer +# fiber_channel_id :integer +# ckb_transaction_id :integer +# created_at :datetime not null +# updated_at :datetime not null +# diff --git a/app/models/referring_cell.rb b/app/models/referring_cell.rb index 9ae25ebf2..2deaba7d1 100644 --- a/app/models/referring_cell.rb +++ b/app/models/referring_cell.rb @@ -56,5 +56,6 @@ def self.create_initial_data_for_ckb_transaction(ckb_transaction) # # Indexes # +# index_referring_cells_on_cell_output_id (cell_output_id) UNIQUE # index_referring_cells_on_contract_id_and_cell_output_id (contract_id,cell_output_id) UNIQUE # diff --git a/app/services/fiber.rb b/app/services/fiber.rb new file mode 100644 index 000000000..660ea0545 --- /dev/null +++ b/app/services/fiber.rb @@ -0,0 +1,43 @@ +class Fiber + include Singleton + + METHOD_NAMES = %w(list_channels).freeze + + def initialize + @id = 0 + @endpoint = ENV["FIBER_RPC_URL"] + end + + METHOD_NAMES.each do |name| + define_method name do |*params| + call_rpc(name, params:) + end + end + + private + + def call_rpc(method, params: []) + @id += 1 + payload = { jsonrpc: "2.0", id: @id, method:, params: } + make_request(@endpoint, payload) + end + + def make_request(endpoint, payload) + response = HTTP.timeout(60).post(endpoint, json: payload) + parse_response(response) + end + + def parse_response(response) + data = JSON.parse(response.to_s) + + return data if data.is_a?(Array) + + if data.is_a?(Hash) + raise ArgumentError, data["error"]["message"] if data["error"].present? + else + raise ArgumentError, "Unexpected response format: #{data.class}" + end + + data + end +end diff --git a/app/workers/fiber_detect_worker.rb b/app/workers/fiber_detect_worker.rb new file mode 100644 index 000000000..818f5f6e2 --- /dev/null +++ b/app/workers/fiber_detect_worker.rb @@ -0,0 +1,6 @@ +class FiberDetectWorker + include Sidekiq::Worker + sidekiq_options queue: "fiber" + + def perform; end +end diff --git a/config/routes/v2.rb b/config/routes/v2.rb index b431ae327..ddfb96913 100644 --- a/config/routes/v2.rb +++ b/config/routes/v2.rb @@ -100,5 +100,9 @@ get :udt_accounts, on: :member end resources :rgb_live_cells, only: :index + namespace :fiber do + resources :peers, only: :index + resources :channels, only: :index + end end end diff --git a/db/migrate/20240918024407_create_fiber_peers.rb b/db/migrate/20240918024407_create_fiber_peers.rb new file mode 100644 index 000000000..bb5aa2f31 --- /dev/null +++ b/db/migrate/20240918024407_create_fiber_peers.rb @@ -0,0 +1,13 @@ +class CreateFiberPeers < ActiveRecord::Migration[7.0] + def change + create_table :fiber_peers do |t| + t.string :name + t.string :peer_id + t.string :rpc_listening_addr + t.datetime :first_channel_opened_at + t.datetime :last_channel_updated_at + + t.timestamps + end + end +end diff --git a/db/migrate/20240918024415_create_fiber_channels.rb b/db/migrate/20240918024415_create_fiber_channels.rb new file mode 100644 index 000000000..7b536f5cf --- /dev/null +++ b/db/migrate/20240918024415_create_fiber_channels.rb @@ -0,0 +1,19 @@ +class CreateFiberChannels < ActiveRecord::Migration[7.0] + def change + create_table :fiber_channels do |t| + t.string :peer_id + t.string :channel_id + t.string :state_name + t.string :state_flags, default: [], array: true + t.decimal :local_balance, precision: 64, scale: 2, default: 0.0 + t.decimal :sent_tlc_balance, precision: 64, scale: 2, default: 0.0 + t.decimal :remote_balance, precision: 64, scale: 2, default: 0.0 + t.decimal :received_tlc_balance, precision: 64, scale: 2, default: 0.0 + t.datetime :shutdown_at + + t.timestamps + end + + add_index :fiber_channels, %i[peer_id channel_id], unique: true + end +end diff --git a/db/migrate/20240918033146_create_fiber_transactions.rb b/db/migrate/20240918033146_create_fiber_transactions.rb new file mode 100644 index 000000000..22f36d455 --- /dev/null +++ b/db/migrate/20240918033146_create_fiber_transactions.rb @@ -0,0 +1,11 @@ +class CreateFiberTransactions < ActiveRecord::Migration[7.0] + def change + create_table :fiber_transactions do |t| + t.integer :fiber_peer_id + t.integer :fiber_channel_id + t.integer :ckb_transaction_id + + t.timestamps + end + end +end diff --git a/db/structure.sql b/db/structure.sql index b0ee52b1a..4f1c42e73 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -23,20 +23,6 @@ CREATE EXTENSION IF NOT EXISTS btree_gin WITH SCHEMA public; COMMENT ON EXTENSION btree_gin IS 'support for indexing common datatypes in GIN'; --- --- Name: pg_buffercache; Type: EXTENSION; Schema: -; Owner: - --- - -CREATE EXTENSION IF NOT EXISTS pg_buffercache WITH SCHEMA public; - - --- --- Name: EXTENSION pg_buffercache; Type: COMMENT; Schema: -; Owner: - --- - -COMMENT ON EXTENSION pg_buffercache IS 'examine the shared buffer cache'; - - -- -- Name: array_subtract(anyarray, anyarray); Type: FUNCTION; Schema: public; Owner: - -- @@ -188,7 +174,7 @@ begin insert into account_books (ckb_transaction_id, address_id) values (row.id, i) ON CONFLICT DO NOTHING; end loop; - END LOOP; + END LOOP; close c; end $$; @@ -210,21 +196,21 @@ DECLARE if new.contained_address_ids is null then new.contained_address_ids := array[]::int[]; end if; - if old is null + if old is null then to_add := new.contained_address_ids; to_remove := array[]::int[]; else - + to_add := array_subtract(new.contained_address_ids, old.contained_address_ids); - to_remove := array_subtract(old.contained_address_ids, new.contained_address_ids); + to_remove := array_subtract(old.contained_address_ids, new.contained_address_ids); end if; if to_add is not null then FOREACH i IN ARRAY to_add - LOOP + LOOP RAISE NOTICE 'ckb_tx_addr_id(%)', i; - insert into account_books (ckb_transaction_id, address_id) + insert into account_books (ckb_transaction_id, address_id) values (new.id, i); END LOOP; end if; @@ -429,9 +415,9 @@ CREATE TABLE public.addresses ( dao_deposit numeric(30,0) DEFAULT 0.0, interest numeric(30,0) DEFAULT 0.0, block_timestamp bigint, - visible boolean DEFAULT true, live_cells_count bigint DEFAULT 0.0, mined_blocks_count integer DEFAULT 0, + visible boolean DEFAULT true, average_deposit_time bigint, unclaimed_compensation numeric(30,0), is_depositor boolean DEFAULT false, @@ -484,6 +470,7 @@ CREATE TABLE public.blocks ( "timestamp" bigint, transactions_root bytea, proposals_hash bytea, + uncles_count integer, extra_hash bytea, uncle_block_hashes bytea, version integer, @@ -510,7 +497,6 @@ CREATE TABLE public.blocks ( nonce numeric(50,0) DEFAULT 0.0, start_number numeric(30,0) DEFAULT 0.0, length numeric(30,0) DEFAULT 0.0, - uncles_count integer, compact_target numeric(20,0), live_cell_changes integer, block_time bigint, @@ -520,8 +506,8 @@ CREATE TABLE public.blocks ( miner_message character varying, extension jsonb, median_timestamp bigint DEFAULT 0.0, - cycles bigint, - ckb_node_version character varying + ckb_node_version character varying, + cycles bigint ); @@ -672,6 +658,39 @@ CREATE SEQUENCE public.bitcoin_statistics_id_seq ALTER SEQUENCE public.bitcoin_statistics_id_seq OWNED BY public.bitcoin_statistics.id; +-- +-- Name: bitcoin_time_locks; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.bitcoin_time_locks ( + id bigint NOT NULL, + bitcoin_transaction_id bigint, + ckb_transaction_id bigint, + cell_output_id bigint, + created_at timestamp(6) without time zone NOT NULL, + updated_at timestamp(6) without time zone NOT NULL +); + + +-- +-- Name: bitcoin_time_locks_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.bitcoin_time_locks_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: bitcoin_time_locks_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.bitcoin_time_locks_id_seq OWNED BY public.bitcoin_time_locks.id; + + -- -- Name: bitcoin_transactions; Type: TABLE; Schema: public; Owner: - -- @@ -1474,9 +1493,9 @@ ALTER SEQUENCE public.contracts_id_seq OWNED BY public.contracts.id; CREATE TABLE public.daily_statistics ( id bigint NOT NULL, - transactions_count character varying DEFAULT '0'::character varying, - addresses_count character varying DEFAULT '0'::character varying, - total_dao_deposit character varying DEFAULT '0.0'::character varying, + transactions_count character varying DEFAULT 0, + addresses_count character varying DEFAULT 0, + total_dao_deposit character varying DEFAULT 0.0, block_timestamp numeric(30,0), created_at_unixtimestamp integer, created_at timestamp(6) without time zone NOT NULL, @@ -1495,8 +1514,8 @@ CREATE TABLE public.daily_statistics ( avg_difficulty character varying DEFAULT '0'::character varying, uncle_rate character varying DEFAULT '0'::character varying, total_depositors_count character varying DEFAULT '0'::character varying, - address_balance_distribution jsonb, total_tx_fee numeric(30,0), + address_balance_distribution jsonb, occupied_capacity numeric(30,0), daily_dao_deposit numeric(30,0), daily_dao_depositors_count integer, @@ -1685,6 +1704,113 @@ CREATE SEQUENCE public.epoch_statistics_id_seq ALTER SEQUENCE public.epoch_statistics_id_seq OWNED BY public.epoch_statistics.id; +-- +-- Name: fiber_channels; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.fiber_channels ( + id bigint NOT NULL, + peer_id character varying, + channel_id character varying, + state_name character varying, + state_flags character varying[] DEFAULT '{}'::character varying[], + local_balance numeric(64,2) DEFAULT 0.0, + sent_tlc_balance numeric(64,2) DEFAULT 0.0, + remote_balance numeric(64,2) DEFAULT 0.0, + received_tlc_balance numeric(64,2) DEFAULT 0.0, + shutdown_at timestamp(6) without time zone, + created_at timestamp(6) without time zone NOT NULL, + updated_at timestamp(6) without time zone NOT NULL +); + + +-- +-- Name: fiber_channels_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.fiber_channels_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: fiber_channels_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.fiber_channels_id_seq OWNED BY public.fiber_channels.id; + + +-- +-- Name: fiber_peers; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.fiber_peers ( + id bigint NOT NULL, + name character varying, + peer_id character varying, + rpc_listening_addr character varying, + first_channel_opened_at timestamp(6) without time zone, + last_channel_updated_at timestamp(6) without time zone, + created_at timestamp(6) without time zone NOT NULL, + updated_at timestamp(6) without time zone NOT NULL +); + + +-- +-- Name: fiber_peers_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.fiber_peers_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: fiber_peers_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.fiber_peers_id_seq OWNED BY public.fiber_peers.id; + + +-- +-- Name: fiber_transactions; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.fiber_transactions ( + id bigint NOT NULL, + fiber_peer_id integer, + fiber_channel_id integer, + ckb_transaction_id integer, + created_at timestamp(6) without time zone NOT NULL, + updated_at timestamp(6) without time zone NOT NULL +); + + +-- +-- Name: fiber_transactions_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.fiber_transactions_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: fiber_transactions_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.fiber_transactions_id_seq OWNED BY public.fiber_transactions.id; + + -- -- Name: forked_blocks; Type: TABLE; Schema: public; Owner: - -- @@ -1697,6 +1823,7 @@ CREATE TABLE public.forked_blocks ( "timestamp" bigint, transactions_root bytea, proposals_hash bytea, + uncles_count integer, extra_hash bytea, uncle_block_hashes bytea, version integer, @@ -1723,7 +1850,6 @@ CREATE TABLE public.forked_blocks ( nonce numeric(50,0) DEFAULT 0.0, start_number numeric(30,0) DEFAULT 0.0, length numeric(30,0) DEFAULT 0.0, - uncles_count integer, compact_target numeric(20,0), live_cell_changes integer, block_time numeric(13,0), @@ -1733,8 +1859,8 @@ CREATE TABLE public.forked_blocks ( miner_message character varying, extension jsonb, median_timestamp numeric DEFAULT 0.0, - cycles bigint, - ckb_node_version character varying + ckb_node_version character varying, + cycles bigint ); @@ -2918,6 +3044,13 @@ ALTER TABLE ONLY public.bitcoin_annotations ALTER COLUMN id SET DEFAULT nextval( ALTER TABLE ONLY public.bitcoin_statistics ALTER COLUMN id SET DEFAULT nextval('public.bitcoin_statistics_id_seq'::regclass); +-- +-- Name: bitcoin_time_locks id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.bitcoin_time_locks ALTER COLUMN id SET DEFAULT nextval('public.bitcoin_time_locks_id_seq'::regclass); + + -- -- Name: bitcoin_transactions id; Type: DEFAULT; Schema: public; Owner: - -- @@ -3058,6 +3191,27 @@ ALTER TABLE ONLY public.deployed_cells ALTER COLUMN id SET DEFAULT nextval('publ ALTER TABLE ONLY public.epoch_statistics ALTER COLUMN id SET DEFAULT nextval('public.epoch_statistics_id_seq'::regclass); +-- +-- Name: fiber_channels id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.fiber_channels ALTER COLUMN id SET DEFAULT nextval('public.fiber_channels_id_seq'::regclass); + + +-- +-- Name: fiber_peers id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.fiber_peers ALTER COLUMN id SET DEFAULT nextval('public.fiber_peers_id_seq'::regclass); + + +-- +-- Name: fiber_transactions id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.fiber_transactions ALTER COLUMN id SET DEFAULT nextval('public.fiber_transactions_id_seq'::regclass); + + -- -- Name: forked_blocks id; Type: DEFAULT; Schema: public; Owner: - -- @@ -3325,6 +3479,14 @@ ALTER TABLE ONLY public.bitcoin_statistics ADD CONSTRAINT bitcoin_statistics_pkey PRIMARY KEY (id); +-- +-- Name: bitcoin_time_locks bitcoin_time_locks_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.bitcoin_time_locks + ADD CONSTRAINT bitcoin_time_locks_pkey PRIMARY KEY (id); + + -- -- Name: bitcoin_transactions bitcoin_transactions_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -3589,6 +3751,30 @@ ALTER TABLE ONLY public.epoch_statistics ADD CONSTRAINT epoch_statistics_pkey PRIMARY KEY (id); +-- +-- Name: fiber_channels fiber_channels_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.fiber_channels + ADD CONSTRAINT fiber_channels_pkey PRIMARY KEY (id); + + +-- +-- Name: fiber_peers fiber_peers_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.fiber_peers + ADD CONSTRAINT fiber_peers_pkey PRIMARY KEY (id); + + +-- +-- Name: fiber_transactions fiber_transactions_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.fiber_transactions + ADD CONSTRAINT fiber_transactions_pkey PRIMARY KEY (id); + + -- -- Name: forked_blocks forked_blocks_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -4462,6 +4648,13 @@ CREATE UNIQUE INDEX index_bitcoin_annotations_on_ckb_transaction_id ON public.bi CREATE UNIQUE INDEX index_bitcoin_statistics_on_timestamp ON public.bitcoin_statistics USING btree ("timestamp"); +-- +-- Name: index_bitcoin_time_locks_on_cell; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX index_bitcoin_time_locks_on_cell ON public.bitcoin_time_locks USING btree (bitcoin_transaction_id, cell_output_id); + + -- -- Name: index_bitcoin_transactions_on_txid; Type: INDEX; Schema: public; Owner: - -- @@ -4735,6 +4928,13 @@ CREATE UNIQUE INDEX index_deployed_cells_on_contract_id_and_cell_output_id ON pu CREATE UNIQUE INDEX index_epoch_statistics_on_epoch_number ON public.epoch_statistics USING btree (epoch_number); +-- +-- Name: index_fiber_channels_on_peer_id_and_channel_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX index_fiber_channels_on_peer_id_and_channel_id ON public.fiber_channels USING btree (peer_id, channel_id); + + -- -- Name: index_forked_events_on_status; Type: INDEX; Schema: public; Owner: - -- @@ -4833,6 +5033,13 @@ CREATE UNIQUE INDEX index_omiga_inscription_infos_on_udt_hash ON public.omiga_in CREATE UNIQUE INDEX index_portfolios_on_user_id_and_address_id ON public.portfolios USING btree (user_id, address_id); +-- +-- Name: index_referring_cells_on_cell_output_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX index_referring_cells_on_cell_output_id ON public.referring_cells USING btree (cell_output_id); + + -- -- Name: index_referring_cells_on_contract_id_and_cell_output_id; Type: INDEX; Schema: public; Owner: - -- @@ -5945,4 +6152,9 @@ INSERT INTO "schema_migrations" (version) VALUES ('20240823071323'), ('20240823071420'), ('20240902025657'), -('20240904043807'); +('20240904043807'), +('20240918024407'), +('20240918024415'), +('20240918033146'); + + diff --git a/lib/scheduler.rb b/lib/scheduler.rb index dec18da08..da0981a17 100644 --- a/lib/scheduler.rb +++ b/lib/scheduler.rb @@ -128,4 +128,8 @@ def call_worker(clz) call_worker GenerateUdtHolderAllocationWorker end +s.every "5m", overlap: false do + call_worker FiberDetectWorker +end + s.join diff --git a/test/factories/fiber_channels.rb b/test/factories/fiber_channels.rb new file mode 100644 index 000000000..7a55b0264 --- /dev/null +++ b/test/factories/fiber_channels.rb @@ -0,0 +1,5 @@ +FactoryBot.define do + factory :fiber_channel do + + end +end diff --git a/test/factories/fiber_peers.rb b/test/factories/fiber_peers.rb new file mode 100644 index 000000000..96b8b3aa3 --- /dev/null +++ b/test/factories/fiber_peers.rb @@ -0,0 +1,5 @@ +FactoryBot.define do + factory :fiber_peer do + + end +end diff --git a/test/factories/fiber_transactions.rb b/test/factories/fiber_transactions.rb new file mode 100644 index 000000000..a0e85493d --- /dev/null +++ b/test/factories/fiber_transactions.rb @@ -0,0 +1,5 @@ +FactoryBot.define do + factory :fiber_transaction do + + end +end diff --git a/test/models/fiber_channel_test.rb b/test/models/fiber_channel_test.rb new file mode 100644 index 000000000..25e751ac1 --- /dev/null +++ b/test/models/fiber_channel_test.rb @@ -0,0 +1,7 @@ +require "test_helper" + +class FiberChannelTest < ActiveSupport::TestCase + # test "the truth" do + # assert true + # end +end diff --git a/test/models/fiber_peer_test.rb b/test/models/fiber_peer_test.rb new file mode 100644 index 000000000..f179352d2 --- /dev/null +++ b/test/models/fiber_peer_test.rb @@ -0,0 +1,7 @@ +require "test_helper" + +class FiberPeerTest < ActiveSupport::TestCase + # test "the truth" do + # assert true + # end +end diff --git a/test/models/fiber_transaction_test.rb b/test/models/fiber_transaction_test.rb new file mode 100644 index 000000000..01f7bde69 --- /dev/null +++ b/test/models/fiber_transaction_test.rb @@ -0,0 +1,7 @@ +require "test_helper" + +class FiberTransactionTest < ActiveSupport::TestCase + # test "the truth" do + # assert true + # end +end From 3ba8b666275b6353837522666ff1ee05508aeb90 Mon Sep 17 00:00:00 2001 From: Rabbit Date: Fri, 20 Sep 2024 08:58:41 +0800 Subject: [PATCH 02/15] feat: add fiber coordinator --- .../api/v2/fiber/channels_controller.rb | 12 +++++ .../api/v2/fiber/channles_controller.rb | 9 ---- .../api/v2/fiber/peers_controller.rb | 37 +++++++++++++++- .../api/v2/portfolio/addresses_controller.rb | 2 +- app/lib/api/v2/exceptions.rb | 20 ++++++++- app/models/fiber_channel.rb | 4 +- app/models/fiber_peer.rb | 12 ++++- app/serializers/fiber_channel_serializer.rb | 7 +++ .../{fiber.rb => fiber_coordinator.rb} | 14 +++--- app/views/api/v2/fiber/channels/show.jbuilder | 3 ++ app/views/api/v2/fiber/peers/index.jbuilder | 6 +++ app/views/api/v2/fiber/peers/show.jbuilder | 8 ++++ app/workers/fiber_detect_worker.rb | 44 ++++++++++++++++++- config/routes/v2.rb | 4 +- config/sidekiq.yml | 1 + 15 files changed, 156 insertions(+), 27 deletions(-) create mode 100644 app/controllers/api/v2/fiber/channels_controller.rb delete mode 100644 app/controllers/api/v2/fiber/channles_controller.rb create mode 100644 app/serializers/fiber_channel_serializer.rb rename app/services/{fiber.rb => fiber_coordinator.rb} (69%) create mode 100644 app/views/api/v2/fiber/channels/show.jbuilder create mode 100644 app/views/api/v2/fiber/peers/index.jbuilder create mode 100644 app/views/api/v2/fiber/peers/show.jbuilder diff --git a/app/controllers/api/v2/fiber/channels_controller.rb b/app/controllers/api/v2/fiber/channels_controller.rb new file mode 100644 index 000000000..bb21847dc --- /dev/null +++ b/app/controllers/api/v2/fiber/channels_controller.rb @@ -0,0 +1,12 @@ +module Api + module V2 + module Fiber + class ChannelsController < BaseController + def show + @channel = FiberChannel.find_by(channel_id: params[:channel_id]) + raise Api::V2::Exceptions::FiberChannelNotFoundError unless @channel + end + end + end + end +end diff --git a/app/controllers/api/v2/fiber/channles_controller.rb b/app/controllers/api/v2/fiber/channles_controller.rb deleted file mode 100644 index ecdee23b3..000000000 --- a/app/controllers/api/v2/fiber/channles_controller.rb +++ /dev/null @@ -1,9 +0,0 @@ -module Api - module V2 - module Fiber - class ChannlesController < BaseController - def index; end - end - end - end -end diff --git a/app/controllers/api/v2/fiber/peers_controller.rb b/app/controllers/api/v2/fiber/peers_controller.rb index 498960d78..b382f77da 100644 --- a/app/controllers/api/v2/fiber/peers_controller.rb +++ b/app/controllers/api/v2/fiber/peers_controller.rb @@ -2,7 +2,42 @@ module Api module V2 module Fiber class PeersController < BaseController - def index; end + before_action :test_connection, only: :create + + def index + @peers = FiberPeer.all + end + + def show + @peer = FiberPeer.find_by(peer_id: params[:peer_id]) + raise Api::V2::Exceptions::FiberPeerNotFoundError unless @peer + end + + def create + fiber_peer = FiberPeer.find_or_initialize_by(peer_id: fiber_peer_params[:peer_id]) + fiber_peer.assign_attributes(fiber_peer_params) + fiber_peer.save! + + FiberDetectWorker.perform_async(fiber_peer.peer_id) + + head :no_content + rescue ActiveRecord::RecordInvalid => e + raise Api::V2::Exceptions::FiberPeerParamsInvalidError.new(e.message) + end + + private + + def fiber_peer_params + params.permit(:name, :peer_id, :rpc_listening_addr) + end + + def test_connection + endpoint = fiber_peer_params[:rpc_listening_addr] + payload = { "peer_id": fiber_peer_params[:peer_id] } + FiberCoordinator.instance.list_channels(endpoint, payload) + rescue ArgumentError => e + raise Api::V2::Exceptions::FiberPeerParamsInvalidError.new(e.message) + end end end end diff --git a/app/controllers/api/v2/portfolio/addresses_controller.rb b/app/controllers/api/v2/portfolio/addresses_controller.rb index 3925d326b..37d5b20fd 100644 --- a/app/controllers/api/v2/portfolio/addresses_controller.rb +++ b/app/controllers/api/v2/portfolio/addresses_controller.rb @@ -10,7 +10,7 @@ def create head :no_content rescue StandardError => e - raise Api::V2::Exceptions::SyncPortfolioAddressesError + raise Api::V2::Exceptions::ParamsInvalidError end end end diff --git a/app/lib/api/v2/exceptions.rb b/app/lib/api/v2/exceptions.rb index 857785075..77c36feef 100644 --- a/app/lib/api/v2/exceptions.rb +++ b/app/lib/api/v2/exceptions.rb @@ -69,7 +69,25 @@ def initialize class AddressNotFoundError < Error def initialize - super code: 2009, status: 404, title: "Address Not Found", detail: "No address found by given address hash or lock hash", href: "" + super code: 2009, status: 404, title: "address Not Found", detail: "No address found by given address hash or lock hash", href: "" + end + end + + class FiberPeerParamsInvalidError < Error + def initialize(detail) + super code: 2010, status: 404, title: "fiber peer params invalid", detail: , href: "" + end + end + + class FiberPeerNotFoundError < Error + def initialize + super code: 2011, status: 404, title: "fiber peer Not Found", detail: "No peer found by given peer_id", href: "" + end + end + + class FiberChannelNotFoundError < Error + def initialize + super code: 2012, status: 404, title: "fiber channel Not Found", detail: "No channel found by given channel_id", href: "" end end end diff --git a/app/models/fiber_channel.rb b/app/models/fiber_channel.rb index 3d48f5cab..1c906e0e2 100644 --- a/app/models/fiber_channel.rb +++ b/app/models/fiber_channel.rb @@ -1,6 +1,6 @@ class FiberChannel < ApplicationRecord - belongs_to :fiber_channel - has_many :fiber_transactions + belongs_to :fiber_peer, class_name: "FiberPeer", foreign_key: :peer_id, primary_key: :peer_id, inverse_of: :fiber_channels + # has_many :fiber_transactions end # == Schema Information diff --git a/app/models/fiber_peer.rb b/app/models/fiber_peer.rb index 4a36bd0db..d58ab537d 100644 --- a/app/models/fiber_peer.rb +++ b/app/models/fiber_peer.rb @@ -1,6 +1,14 @@ class FiberPeer < ApplicationRecord - has_many :fiber_channels, dependent: :destroy, foreign_key: :peer_id - has_many :fiber_transactions + has_many :fiber_channels, foreign_key: :peer_id, primary_key: :peer_id, inverse_of: :fiber_peer, dependent: :destroy + # has_many :fiber_transactions + + def total_local_balance + fiber_channels.where(state_name: "CHANNEL_READY").sum(:local_balance) + end + + def channels_count + fiber_channels.where(state_name: "CHANNEL_READY").count + end end # == Schema Information diff --git a/app/serializers/fiber_channel_serializer.rb b/app/serializers/fiber_channel_serializer.rb new file mode 100644 index 000000000..2262ef5a6 --- /dev/null +++ b/app/serializers/fiber_channel_serializer.rb @@ -0,0 +1,7 @@ +class FiberChannelSerializer + include FastJsonapi::ObjectSerializer + + attributes :peer_id, :channel_id, :state_name, :state_flags, :local_balance, + :sent_tlc_balance, :remote_balance, :received_tlc_balance, :shutdown_at, + :created_at, :updated_at +end diff --git a/app/services/fiber.rb b/app/services/fiber_coordinator.rb similarity index 69% rename from app/services/fiber.rb rename to app/services/fiber_coordinator.rb index 660ea0545..e36e19ba4 100644 --- a/app/services/fiber.rb +++ b/app/services/fiber_coordinator.rb @@ -1,25 +1,23 @@ -class Fiber +class FiberCoordinator include Singleton - METHOD_NAMES = %w(list_channels).freeze def initialize @id = 0 - @endpoint = ENV["FIBER_RPC_URL"] end METHOD_NAMES.each do |name| - define_method name do |*params| - call_rpc(name, params:) + define_method name do |endpoint, *params| + call_rpc(name, endpoint, params:) end end private - def call_rpc(method, params: []) + def call_rpc(method, endpoint, params: []) @id += 1 payload = { jsonrpc: "2.0", id: @id, method:, params: } - make_request(@endpoint, payload) + make_request(endpoint, payload) end def make_request(endpoint, payload) @@ -33,7 +31,7 @@ def parse_response(response) return data if data.is_a?(Array) if data.is_a?(Hash) - raise ArgumentError, data["error"]["message"] if data["error"].present? + raise ArgumentError, data["error"]["data"] if data["error"].present? else raise ArgumentError, "Unexpected response format: #{data.class}" end diff --git a/app/views/api/v2/fiber/channels/show.jbuilder b/app/views/api/v2/fiber/channels/show.jbuilder new file mode 100644 index 000000000..201dd9d28 --- /dev/null +++ b/app/views/api/v2/fiber/channels/show.jbuilder @@ -0,0 +1,3 @@ +json.data do + json.(@channel, :channel_id, :state_name, :state_flags, :shutdown_at, :created_at, :updated_at, :local_balance, :sent_tlc_balance, :remote_balance, :received_tlc_balance) +end diff --git a/app/views/api/v2/fiber/peers/index.jbuilder b/app/views/api/v2/fiber/peers/index.jbuilder new file mode 100644 index 000000000..c0dfce2e0 --- /dev/null +++ b/app/views/api/v2/fiber/peers/index.jbuilder @@ -0,0 +1,6 @@ +json.data do + json.fiber_peers @peers do |peer| + json.(peer, :name, :peer_id, :rpc_listening_addr, :first_channel_opened_at,:last_channel_updated_at, :channels_count) + json.total_local_balance peer.total_local_balance.to_s + end +end \ No newline at end of file diff --git a/app/views/api/v2/fiber/peers/show.jbuilder b/app/views/api/v2/fiber/peers/show.jbuilder new file mode 100644 index 000000000..2946b53ce --- /dev/null +++ b/app/views/api/v2/fiber/peers/show.jbuilder @@ -0,0 +1,8 @@ +json.data do + json.(@peer, :peer_id, :rpc_listening_addr, :first_channel_opened_at, :last_channel_updated_at) + json.fiber_channels @peer.fiber_channels do |peer_channel| + json.channel_id peer_channel.channel_id + json.state_name peer_channel.state_name + json.state_flags peer_channel.state_flags + end +end diff --git a/app/workers/fiber_detect_worker.rb b/app/workers/fiber_detect_worker.rb index 818f5f6e2..d624f2512 100644 --- a/app/workers/fiber_detect_worker.rb +++ b/app/workers/fiber_detect_worker.rb @@ -2,5 +2,47 @@ class FiberDetectWorker include Sidekiq::Worker sidekiq_options queue: "fiber" - def perform; end + def perform(peer_id) + fiber_peers = peer_id.present? ? FiberPeer.where(peer_id:) : FiberPeer.all + fiber_peers.each { sync_with_fiber_channels(_1) } + end + + private + + def sync_with_fiber_channels(fiber_peer) + channels_attributes = build_channels_attributes(fiber_peer) + FiberChannel.upsert_all(channels_attributes, unique_by: %i[peer_id channel_id]) + end + + def build_channels_attributes(fiber_peer) + data = rpc.list_channels(fiber_peer.rpc_listening_addr, { peer_id: fiber_peer.peer_id }) + data["result"]["channels"].map do |channel| + { + peer_id: channel["peer_id"], + channel_id: channel["channel_id"], + state_name: channel["state"]["state_name"], + state_flags: parse_state_flags(channel["state"]["state_flags"]), + local_balance: channel["local_balance"].to_i(16), + sent_tlc_balance: channel["sent_tlc_balance"].to_i(16), + remote_balance: channel["remote_balance"].to_i(16), + received_tlc_balance: channel["received_tlc_balance"].to_i(16), + created_at: Time.at(channel["created_at"].to_i(16) / 10**6), + } + end + end + + def parse_state_flags(flags) + case flags + when Array + flags + when String + flags.split("|") + else + [] + end + end + + def rpc + @rpc ||= FiberCoordinator.instance + end end diff --git a/config/routes/v2.rb b/config/routes/v2.rb index ddfb96913..b0cbf8fcc 100644 --- a/config/routes/v2.rb +++ b/config/routes/v2.rb @@ -101,8 +101,8 @@ end resources :rgb_live_cells, only: :index namespace :fiber do - resources :peers, only: :index - resources :channels, only: :index + resources :peers, param: :peer_id, only: %i[index show create] + resources :channels, param: :channel_id, only: :show end end end diff --git a/config/sidekiq.yml b/config/sidekiq.yml index eb0df0b76..7054f09af 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -5,6 +5,7 @@ - critical - default - bitcoin + - fiber - low production: From 0283caa755a0aededc637b255fa74b28d1efa35a Mon Sep 17 00:00:00 2001 From: Rabbit Date: Fri, 20 Sep 2024 18:09:01 +0800 Subject: [PATCH 03/15] chore: add foreign key fiber_peer_id --- app/controllers/api/v2/fiber/peers_controller.rb | 8 +++----- app/lib/api/v2/exceptions.rb | 2 +- app/models/fiber_channel.rb | 4 +++- app/models/fiber_peer.rb | 2 +- app/views/api/v2/fiber/peers/index.jbuilder | 2 +- app/workers/fiber_detect_worker.rb | 1 + config/routes/v2.rb | 2 +- ...920094807_add_fiber_peer_id_to_fiber_channels.rb | 6 ++++++ db/structure.sql | 13 +++++++++++-- 9 files changed, 28 insertions(+), 12 deletions(-) create mode 100644 db/migrate/20240920094807_add_fiber_peer_id_to_fiber_channels.rb diff --git a/app/controllers/api/v2/fiber/peers_controller.rb b/app/controllers/api/v2/fiber/peers_controller.rb index b382f77da..61547b92e 100644 --- a/app/controllers/api/v2/fiber/peers_controller.rb +++ b/app/controllers/api/v2/fiber/peers_controller.rb @@ -9,7 +9,7 @@ def index end def show - @peer = FiberPeer.find_by(peer_id: params[:peer_id]) + @peer = FiberPeer.find_by(id: params[:id]) raise Api::V2::Exceptions::FiberPeerNotFoundError unless @peer end @@ -32,10 +32,8 @@ def fiber_peer_params end def test_connection - endpoint = fiber_peer_params[:rpc_listening_addr] - payload = { "peer_id": fiber_peer_params[:peer_id] } - FiberCoordinator.instance.list_channels(endpoint, payload) - rescue ArgumentError => e + FiberCoordinator.instance.list_channels(fiber_peer_params[:rpc_listening_addr], { "peer_id": nil }) + rescue StandardError => e raise Api::V2::Exceptions::FiberPeerParamsInvalidError.new(e.message) end end diff --git a/app/lib/api/v2/exceptions.rb b/app/lib/api/v2/exceptions.rb index 77c36feef..e9a9642cb 100644 --- a/app/lib/api/v2/exceptions.rb +++ b/app/lib/api/v2/exceptions.rb @@ -81,7 +81,7 @@ def initialize(detail) class FiberPeerNotFoundError < Error def initialize - super code: 2011, status: 404, title: "fiber peer Not Found", detail: "No peer found by given peer_id", href: "" + super code: 2011, status: 404, title: "fiber peer Not Found", detail: "No peer found by given peer id", href: "" end end diff --git a/app/models/fiber_channel.rb b/app/models/fiber_channel.rb index 1c906e0e2..5b16ea5ce 100644 --- a/app/models/fiber_channel.rb +++ b/app/models/fiber_channel.rb @@ -1,5 +1,5 @@ class FiberChannel < ApplicationRecord - belongs_to :fiber_peer, class_name: "FiberPeer", foreign_key: :peer_id, primary_key: :peer_id, inverse_of: :fiber_channels + belongs_to :fiber_peer # has_many :fiber_transactions end @@ -19,8 +19,10 @@ class FiberChannel < ApplicationRecord # shutdown_at :datetime # created_at :datetime not null # updated_at :datetime not null +# fiber_peer_id :integer # # Indexes # +# index_fiber_channels_on_fiber_peer_id (fiber_peer_id) # index_fiber_channels_on_peer_id_and_channel_id (peer_id,channel_id) UNIQUE # diff --git a/app/models/fiber_peer.rb b/app/models/fiber_peer.rb index d58ab537d..35e040818 100644 --- a/app/models/fiber_peer.rb +++ b/app/models/fiber_peer.rb @@ -1,5 +1,5 @@ class FiberPeer < ApplicationRecord - has_many :fiber_channels, foreign_key: :peer_id, primary_key: :peer_id, inverse_of: :fiber_peer, dependent: :destroy + has_many :fiber_channels, dependent: :destroy # has_many :fiber_transactions def total_local_balance diff --git a/app/views/api/v2/fiber/peers/index.jbuilder b/app/views/api/v2/fiber/peers/index.jbuilder index c0dfce2e0..756fa5efb 100644 --- a/app/views/api/v2/fiber/peers/index.jbuilder +++ b/app/views/api/v2/fiber/peers/index.jbuilder @@ -1,6 +1,6 @@ json.data do json.fiber_peers @peers do |peer| - json.(peer, :name, :peer_id, :rpc_listening_addr, :first_channel_opened_at,:last_channel_updated_at, :channels_count) + json.(peer, :id, :name, :peer_id, :rpc_listening_addr, :first_channel_opened_at,:last_channel_updated_at, :channels_count) json.total_local_balance peer.total_local_balance.to_s end end \ No newline at end of file diff --git a/app/workers/fiber_detect_worker.rb b/app/workers/fiber_detect_worker.rb index d624f2512..9b8f4544f 100644 --- a/app/workers/fiber_detect_worker.rb +++ b/app/workers/fiber_detect_worker.rb @@ -18,6 +18,7 @@ def build_channels_attributes(fiber_peer) data = rpc.list_channels(fiber_peer.rpc_listening_addr, { peer_id: fiber_peer.peer_id }) data["result"]["channels"].map do |channel| { + fiber_peer_id: fiber_peer.id, peer_id: channel["peer_id"], channel_id: channel["channel_id"], state_name: channel["state"]["state_name"], diff --git a/config/routes/v2.rb b/config/routes/v2.rb index b0cbf8fcc..446cf6cd6 100644 --- a/config/routes/v2.rb +++ b/config/routes/v2.rb @@ -101,7 +101,7 @@ end resources :rgb_live_cells, only: :index namespace :fiber do - resources :peers, param: :peer_id, only: %i[index show create] + resources :peers, only: %i[index show create] resources :channels, param: :channel_id, only: :show end end diff --git a/db/migrate/20240920094807_add_fiber_peer_id_to_fiber_channels.rb b/db/migrate/20240920094807_add_fiber_peer_id_to_fiber_channels.rb new file mode 100644 index 000000000..f80e003ad --- /dev/null +++ b/db/migrate/20240920094807_add_fiber_peer_id_to_fiber_channels.rb @@ -0,0 +1,6 @@ +class AddFiberPeerIdToFiberChannels < ActiveRecord::Migration[7.0] + def change + add_column :fiber_channels, :fiber_peer_id, :integer + add_index :fiber_channels, :fiber_peer_id + end +end diff --git a/db/structure.sql b/db/structure.sql index 4f1c42e73..ade3e0627 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -1720,7 +1720,8 @@ CREATE TABLE public.fiber_channels ( received_tlc_balance numeric(64,2) DEFAULT 0.0, shutdown_at timestamp(6) without time zone, created_at timestamp(6) without time zone NOT NULL, - updated_at timestamp(6) without time zone NOT NULL + updated_at timestamp(6) without time zone NOT NULL, + fiber_peer_id integer ); @@ -4928,6 +4929,13 @@ CREATE UNIQUE INDEX index_deployed_cells_on_contract_id_and_cell_output_id ON pu CREATE UNIQUE INDEX index_epoch_statistics_on_epoch_number ON public.epoch_statistics USING btree (epoch_number); +-- +-- Name: index_fiber_channels_on_fiber_peer_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_fiber_channels_on_fiber_peer_id ON public.fiber_channels USING btree (fiber_peer_id); + + -- -- Name: index_fiber_channels_on_peer_id_and_channel_id; Type: INDEX; Schema: public; Owner: - -- @@ -6155,6 +6163,7 @@ INSERT INTO "schema_migrations" (version) VALUES ('20240904043807'), ('20240918024407'), ('20240918024415'), -('20240918033146'); +('20240918033146'), +('20240920094807'); From ec741f073fc042608f3df8148be3614fd5856c2c Mon Sep 17 00:00:00 2001 From: Rabbit Date: Fri, 20 Sep 2024 18:39:15 +0800 Subject: [PATCH 04/15] chore: rename sent_tlc_balance column --- app/controllers/api/v2/fiber/peers_controller.rb | 2 +- app/models/fiber_channel.rb | 2 +- app/models/fiber_peer.rb | 2 ++ app/views/api/v2/fiber/peers/index.jbuilder | 2 +- app/views/api/v2/fiber/peers/show.jbuilder | 1 + app/workers/fiber_detect_worker.rb | 4 ++-- config/routes/v2.rb | 2 +- .../20240920094807_add_fiber_peer_id_to_fiber_channels.rb | 1 + db/structure.sql | 2 +- 9 files changed, 11 insertions(+), 7 deletions(-) diff --git a/app/controllers/api/v2/fiber/peers_controller.rb b/app/controllers/api/v2/fiber/peers_controller.rb index 61547b92e..1fb3b8731 100644 --- a/app/controllers/api/v2/fiber/peers_controller.rb +++ b/app/controllers/api/v2/fiber/peers_controller.rb @@ -9,7 +9,7 @@ def index end def show - @peer = FiberPeer.find_by(id: params[:id]) + @peer = FiberPeer.find_by(peer_id: params[:peer_id]) raise Api::V2::Exceptions::FiberPeerNotFoundError unless @peer end diff --git a/app/models/fiber_channel.rb b/app/models/fiber_channel.rb index 5b16ea5ce..5db6d0f6e 100644 --- a/app/models/fiber_channel.rb +++ b/app/models/fiber_channel.rb @@ -13,7 +13,7 @@ class FiberChannel < ApplicationRecord # state_name :string # state_flags :string default([]), is an Array # local_balance :decimal(64, 2) default(0.0) -# sent_tlc_balance :decimal(64, 2) default(0.0) +# offered_tlc_balance :decimal(64, 2) default(0.0) # remote_balance :decimal(64, 2) default(0.0) # received_tlc_balance :decimal(64, 2) default(0.0) # shutdown_at :datetime diff --git a/app/models/fiber_peer.rb b/app/models/fiber_peer.rb index 35e040818..8843251c9 100644 --- a/app/models/fiber_peer.rb +++ b/app/models/fiber_peer.rb @@ -2,6 +2,8 @@ class FiberPeer < ApplicationRecord has_many :fiber_channels, dependent: :destroy # has_many :fiber_transactions + validates :peer_id, presence: true, uniqueness: true + def total_local_balance fiber_channels.where(state_name: "CHANNEL_READY").sum(:local_balance) end diff --git a/app/views/api/v2/fiber/peers/index.jbuilder b/app/views/api/v2/fiber/peers/index.jbuilder index 756fa5efb..c0dfce2e0 100644 --- a/app/views/api/v2/fiber/peers/index.jbuilder +++ b/app/views/api/v2/fiber/peers/index.jbuilder @@ -1,6 +1,6 @@ json.data do json.fiber_peers @peers do |peer| - json.(peer, :id, :name, :peer_id, :rpc_listening_addr, :first_channel_opened_at,:last_channel_updated_at, :channels_count) + json.(peer, :name, :peer_id, :rpc_listening_addr, :first_channel_opened_at,:last_channel_updated_at, :channels_count) json.total_local_balance peer.total_local_balance.to_s end end \ No newline at end of file diff --git a/app/views/api/v2/fiber/peers/show.jbuilder b/app/views/api/v2/fiber/peers/show.jbuilder index 2946b53ce..2ea20791f 100644 --- a/app/views/api/v2/fiber/peers/show.jbuilder +++ b/app/views/api/v2/fiber/peers/show.jbuilder @@ -1,6 +1,7 @@ json.data do json.(@peer, :peer_id, :rpc_listening_addr, :first_channel_opened_at, :last_channel_updated_at) json.fiber_channels @peer.fiber_channels do |peer_channel| + json.peer_id peer_channel.peer_id json.channel_id peer_channel.channel_id json.state_name peer_channel.state_name json.state_flags peer_channel.state_flags diff --git a/app/workers/fiber_detect_worker.rb b/app/workers/fiber_detect_worker.rb index 9b8f4544f..ee8267748 100644 --- a/app/workers/fiber_detect_worker.rb +++ b/app/workers/fiber_detect_worker.rb @@ -15,7 +15,7 @@ def sync_with_fiber_channels(fiber_peer) end def build_channels_attributes(fiber_peer) - data = rpc.list_channels(fiber_peer.rpc_listening_addr, { peer_id: fiber_peer.peer_id }) + data = rpc.list_channels(fiber_peer.rpc_listening_addr, { peer_id: nil }) data["result"]["channels"].map do |channel| { fiber_peer_id: fiber_peer.id, @@ -24,7 +24,7 @@ def build_channels_attributes(fiber_peer) state_name: channel["state"]["state_name"], state_flags: parse_state_flags(channel["state"]["state_flags"]), local_balance: channel["local_balance"].to_i(16), - sent_tlc_balance: channel["sent_tlc_balance"].to_i(16), + offered_tlc_balance: channel["offered_tlc_balance"].to_i(16), remote_balance: channel["remote_balance"].to_i(16), received_tlc_balance: channel["received_tlc_balance"].to_i(16), created_at: Time.at(channel["created_at"].to_i(16) / 10**6), diff --git a/config/routes/v2.rb b/config/routes/v2.rb index 446cf6cd6..b0cbf8fcc 100644 --- a/config/routes/v2.rb +++ b/config/routes/v2.rb @@ -101,7 +101,7 @@ end resources :rgb_live_cells, only: :index namespace :fiber do - resources :peers, only: %i[index show create] + resources :peers, param: :peer_id, only: %i[index show create] resources :channels, param: :channel_id, only: :show end end diff --git a/db/migrate/20240920094807_add_fiber_peer_id_to_fiber_channels.rb b/db/migrate/20240920094807_add_fiber_peer_id_to_fiber_channels.rb index f80e003ad..375d2683d 100644 --- a/db/migrate/20240920094807_add_fiber_peer_id_to_fiber_channels.rb +++ b/db/migrate/20240920094807_add_fiber_peer_id_to_fiber_channels.rb @@ -1,5 +1,6 @@ class AddFiberPeerIdToFiberChannels < ActiveRecord::Migration[7.0] def change + rename_column :fiber_channels, :sent_tlc_balance, :offered_tlc_balance add_column :fiber_channels, :fiber_peer_id, :integer add_index :fiber_channels, :fiber_peer_id end diff --git a/db/structure.sql b/db/structure.sql index ade3e0627..c6e7b4b0d 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -1715,7 +1715,7 @@ CREATE TABLE public.fiber_channels ( state_name character varying, state_flags character varying[] DEFAULT '{}'::character varying[], local_balance numeric(64,2) DEFAULT 0.0, - sent_tlc_balance numeric(64,2) DEFAULT 0.0, + offered_tlc_balance numeric(64,2) DEFAULT 0.0, remote_balance numeric(64,2) DEFAULT 0.0, received_tlc_balance numeric(64,2) DEFAULT 0.0, shutdown_at timestamp(6) without time zone, From 6b5047c50967d1f7acd10d17be493fc4091766e8 Mon Sep 17 00:00:00 2001 From: Rabbit Date: Fri, 20 Sep 2024 18:56:48 +0800 Subject: [PATCH 05/15] chore: refine fiber channel jbuilder attribute definitions --- app/views/api/v2/fiber/channels/show.jbuilder | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/api/v2/fiber/channels/show.jbuilder b/app/views/api/v2/fiber/channels/show.jbuilder index 201dd9d28..dea79e2c4 100644 --- a/app/views/api/v2/fiber/channels/show.jbuilder +++ b/app/views/api/v2/fiber/channels/show.jbuilder @@ -1,3 +1,3 @@ json.data do - json.(@channel, :channel_id, :state_name, :state_flags, :shutdown_at, :created_at, :updated_at, :local_balance, :sent_tlc_balance, :remote_balance, :received_tlc_balance) + json.(@channel, :channel_id, :state_name, :state_flags, :shutdown_at, :created_at, :updated_at, :local_balance, :offered_tlc_balance, :remote_balance, :received_tlc_balance) end From 20ee3bee792389784ced4b7cc8a2ff64c032b399 Mon Sep 17 00:00:00 2001 From: Rabbit Date: Mon, 23 Sep 2024 16:15:57 +0800 Subject: [PATCH 06/15] feat: fiber peers paginate --- app/controllers/api/v2/fiber/peers_controller.rb | 4 +++- app/models/fiber_peer.rb | 5 +++++ app/views/api/v2/fiber/peers/index.jbuilder | 6 +++++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/v2/fiber/peers_controller.rb b/app/controllers/api/v2/fiber/peers_controller.rb index 1fb3b8731..4c59829a2 100644 --- a/app/controllers/api/v2/fiber/peers_controller.rb +++ b/app/controllers/api/v2/fiber/peers_controller.rb @@ -5,7 +5,9 @@ class PeersController < BaseController before_action :test_connection, only: :create def index - @peers = FiberPeer.all + @page = params.fetch(:page, 1) + @page_size = params.fetch(:page_size, FiberPeer.default_per_page) + @peers = FiberPeer.all.page(@page).per(@page_size) end def show diff --git a/app/models/fiber_peer.rb b/app/models/fiber_peer.rb index 8843251c9..f24b0b943 100644 --- a/app/models/fiber_peer.rb +++ b/app/models/fiber_peer.rb @@ -1,4 +1,9 @@ class FiberPeer < ApplicationRecord + MAX_PAGINATES_PER = 100 + DEFAULT_PAGINATES_PER = 10 + paginates_per DEFAULT_PAGINATES_PER + max_paginates_per MAX_PAGINATES_PER + has_many :fiber_channels, dependent: :destroy # has_many :fiber_transactions diff --git a/app/views/api/v2/fiber/peers/index.jbuilder b/app/views/api/v2/fiber/peers/index.jbuilder index c0dfce2e0..2566792ca 100644 --- a/app/views/api/v2/fiber/peers/index.jbuilder +++ b/app/views/api/v2/fiber/peers/index.jbuilder @@ -3,4 +3,8 @@ json.data do json.(peer, :name, :peer_id, :rpc_listening_addr, :first_channel_opened_at,:last_channel_updated_at, :channels_count) json.total_local_balance peer.total_local_balance.to_s end -end \ No newline at end of file +end +json.meta do + json.total @peers.total_count + json.page_size @page_size.to_i +end From 7345eba8fe608b52840772ada01aba374f5c4bc1 Mon Sep 17 00:00:00 2001 From: Rabbit Date: Tue, 24 Sep 2024 16:14:00 +0800 Subject: [PATCH 07/15] chore: change rpc_listening_addr type to array --- app/controllers/api/v2/fiber/peers_controller.rb | 4 +++- app/models/fiber_channel.rb | 8 ++++++++ app/models/fiber_peer.rb | 2 +- app/views/api/v2/fiber/channels/show.jbuilder | 8 ++++++++ .../20240924065539_update_rpc_listening_addr_to_array.rb | 5 +++++ db/structure.sql | 5 +++-- 6 files changed, 28 insertions(+), 4 deletions(-) create mode 100644 db/migrate/20240924065539_update_rpc_listening_addr_to_array.rb diff --git a/app/controllers/api/v2/fiber/peers_controller.rb b/app/controllers/api/v2/fiber/peers_controller.rb index 4c59829a2..fb1584b01 100644 --- a/app/controllers/api/v2/fiber/peers_controller.rb +++ b/app/controllers/api/v2/fiber/peers_controller.rb @@ -17,7 +17,9 @@ def show def create fiber_peer = FiberPeer.find_or_initialize_by(peer_id: fiber_peer_params[:peer_id]) - fiber_peer.assign_attributes(fiber_peer_params) + fiber_peer.name = fiber_peer_params[:name] + new_rpc = Array(fiber_peer_params[:rpc_listening_addr]) + fiber_peer.rpc_listening_addr = (fiber_peer.rpc_listening_addr + new_rpc).uniq fiber_peer.save! FiberDetectWorker.perform_async(fiber_peer.peer_id) diff --git a/app/models/fiber_channel.rb b/app/models/fiber_channel.rb index 5db6d0f6e..6272408b2 100644 --- a/app/models/fiber_channel.rb +++ b/app/models/fiber_channel.rb @@ -1,6 +1,14 @@ class FiberChannel < ApplicationRecord belongs_to :fiber_peer # has_many :fiber_transactions + + def local_peer + fiber_peer || FiberPeer.new + end + + def remote_peer + FiberPeer.find_by(peer_id:) || FiberPeer.new + end end # == Schema Information diff --git a/app/models/fiber_peer.rb b/app/models/fiber_peer.rb index f24b0b943..9edd9bbde 100644 --- a/app/models/fiber_peer.rb +++ b/app/models/fiber_peer.rb @@ -25,7 +25,7 @@ def channels_count # id :bigint not null, primary key # name :string # peer_id :string -# rpc_listening_addr :string +# rpc_listening_addr :string default([]), is an Array # first_channel_opened_at :datetime # last_channel_updated_at :datetime # created_at :datetime not null diff --git a/app/views/api/v2/fiber/channels/show.jbuilder b/app/views/api/v2/fiber/channels/show.jbuilder index dea79e2c4..570f174f0 100644 --- a/app/views/api/v2/fiber/channels/show.jbuilder +++ b/app/views/api/v2/fiber/channels/show.jbuilder @@ -1,3 +1,11 @@ json.data do json.(@channel, :channel_id, :state_name, :state_flags, :shutdown_at, :created_at, :updated_at, :local_balance, :offered_tlc_balance, :remote_balance, :received_tlc_balance) + + json.local_peer do + json.(@channel.local_peer, :peer_id, :name, :rpc_listening_addr) + end + + json.remote_peer do + json.(@channel.remote_peer, :peer_id, :name, :rpc_listening_addr) + end end diff --git a/db/migrate/20240924065539_update_rpc_listening_addr_to_array.rb b/db/migrate/20240924065539_update_rpc_listening_addr_to_array.rb new file mode 100644 index 000000000..d70a90ca4 --- /dev/null +++ b/db/migrate/20240924065539_update_rpc_listening_addr_to_array.rb @@ -0,0 +1,5 @@ +class UpdateRpcListeningAddrToArray < ActiveRecord::Migration[7.0] + def change + change_column :fiber_peers, :rpc_listening_addr, :string, array: true, default: [], using: "(string_to_array(rpc_listening_addr, ','))" + end +end diff --git a/db/structure.sql b/db/structure.sql index c6e7b4b0d..8fb54337e 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -1752,7 +1752,7 @@ CREATE TABLE public.fiber_peers ( id bigint NOT NULL, name character varying, peer_id character varying, - rpc_listening_addr character varying, + rpc_listening_addr character varying[] DEFAULT '{}'::character varying[], first_channel_opened_at timestamp(6) without time zone, last_channel_updated_at timestamp(6) without time zone, created_at timestamp(6) without time zone NOT NULL, @@ -6164,6 +6164,7 @@ INSERT INTO "schema_migrations" (version) VALUES ('20240918024407'), ('20240918024415'), ('20240918033146'), -('20240920094807'); +('20240920094807'), +('20240924065539'); From 76896759b10129ea59b62ebf25559f6ba6236b2d Mon Sep 17 00:00:00 2001 From: Rabbit Date: Mon, 14 Oct 2024 13:53:15 +0800 Subject: [PATCH 08/15] feat: sync fiber graph nodes and channels --- .env.example | 3 + .../api/v2/fiber/graph_channels_controller.rb | 13 ++ .../api/v2/fiber/graph_nodes_controller.rb | 22 +++ app/lib/api/v2/exceptions.rb | 16 ++- app/models/fiber_channel.rb | 1 - app/models/fiber_graph_channel.rb | 30 ++++ app/models/fiber_graph_node.rb | 25 ++++ app/models/fiber_peer.rb | 7 +- app/services/fiber_coordinator.rb | 2 +- .../v2/fiber/graph_channels/index.jbuilder | 16 +++ .../api/v2/fiber/graph_nodes/index.jbuilder | 11 ++ .../api/v2/fiber/graph_nodes/show.jbuilder | 16 +++ app/workers/fiber_detect_worker.rb | 2 +- app/workers/fiber_graph_detect_worker.rb | 78 ++++++++++ config/routes/v2.rb | 2 + ...20241012014906_create_fiber_graph_infos.rb | 33 +++++ db/structure.sql | 136 +++++++++++++++++- lib/scheduler.rb | 3 +- test/factories/fiber_graph_channels.rb | 5 + ...r_transactions.rb => fiber_graph_nodes.rb} | 2 +- test/models/fiber_graph_channel_test.rb | 7 + ...ction_test.rb => fiber_graph_node_test.rb} | 2 +- 22 files changed, 418 insertions(+), 14 deletions(-) create mode 100644 app/controllers/api/v2/fiber/graph_channels_controller.rb create mode 100644 app/controllers/api/v2/fiber/graph_nodes_controller.rb create mode 100644 app/models/fiber_graph_channel.rb create mode 100644 app/models/fiber_graph_node.rb create mode 100644 app/views/api/v2/fiber/graph_channels/index.jbuilder create mode 100644 app/views/api/v2/fiber/graph_nodes/index.jbuilder create mode 100644 app/views/api/v2/fiber/graph_nodes/show.jbuilder create mode 100644 app/workers/fiber_graph_detect_worker.rb create mode 100644 db/migrate/20241012014906_create_fiber_graph_infos.rb create mode 100644 test/factories/fiber_graph_channels.rb rename test/factories/{fiber_transactions.rb => fiber_graph_nodes.rb} (52%) create mode 100644 test/models/fiber_graph_channel_test.rb rename test/models/{fiber_transaction_test.rb => fiber_graph_node_test.rb} (59%) diff --git a/.env.example b/.env.example index fe6ed13b7..a468ee942 100644 --- a/.env.example +++ b/.env.example @@ -127,3 +127,6 @@ BITCOIN_SIGNET_PASS="" # Dynamic CORS configuration PARTNER_DOMAINS="/localhost:\d*/" + +# -------------------------------- Fiber segment -------------------------------- +FIBER_NODE_URL="" \ No newline at end of file diff --git a/app/controllers/api/v2/fiber/graph_channels_controller.rb b/app/controllers/api/v2/fiber/graph_channels_controller.rb new file mode 100644 index 000000000..4e6108b0b --- /dev/null +++ b/app/controllers/api/v2/fiber/graph_channels_controller.rb @@ -0,0 +1,13 @@ +module Api + module V2 + module Fiber + class GraphChannelsController < BaseController + def index + @page = params.fetch(:page, 1) + @page_size = params.fetch(:page_size, FiberPeer.default_per_page) + @channels = FiberGraphChannel.all.page(@page).per(@page_size) + end + end + end + end +end diff --git a/app/controllers/api/v2/fiber/graph_nodes_controller.rb b/app/controllers/api/v2/fiber/graph_nodes_controller.rb new file mode 100644 index 000000000..60fd220f3 --- /dev/null +++ b/app/controllers/api/v2/fiber/graph_nodes_controller.rb @@ -0,0 +1,22 @@ +module Api + module V2 + module Fiber + class GraphNodesController < BaseController + def index + @page = params.fetch(:page, 1) + @page_size = params.fetch(:page_size, FiberGraphNode.default_per_page) + @nodes = FiberGraphNode.all.page(@page).per(@page_size) + end + + def show + @node = FiberGraphNode.find_by(node_id: params[:node_id]) + raise Api::V2::Exceptions::FiberGraphNodeNotFoundError unless @node + + @graph_channels = FiberGraphChannel.where(node1: params[:node_id]).or( + FiberGraphChannel.where(node2: params[:node_id]), + ) + end + end + end + end +end diff --git a/app/lib/api/v2/exceptions.rb b/app/lib/api/v2/exceptions.rb index e9a9642cb..c33080b83 100644 --- a/app/lib/api/v2/exceptions.rb +++ b/app/lib/api/v2/exceptions.rb @@ -21,13 +21,13 @@ def initialize(detail) class TokenCollectionNotFoundError < Error def initialize - super(code: 2001, status: 404, title: "Token Collection Not Found", detail: "No token collection found by given script hash or id", href: "") + super(code: 2001, status: 404, title: "token collection not found", detail: "No token collection found by given script hash or id", href: "") end end class AddressNotMatchEnvironmentError < Error def initialize(ckb_net_mode) - super(code: 2022, status: 422, title: "Address is invalid", detail: "This address is not the #{ckb_net_mode} address", href: "") + super(code: 2022, status: 422, title: "address is invalid", detail: "This address is not the #{ckb_net_mode} address", href: "") end end @@ -69,7 +69,7 @@ def initialize class AddressNotFoundError < Error def initialize - super code: 2009, status: 404, title: "address Not Found", detail: "No address found by given address hash or lock hash", href: "" + super code: 2009, status: 404, title: "address not found", detail: "No address found by given address hash or lock hash", href: "" end end @@ -81,13 +81,19 @@ def initialize(detail) class FiberPeerNotFoundError < Error def initialize - super code: 2011, status: 404, title: "fiber peer Not Found", detail: "No peer found by given peer id", href: "" + super code: 2011, status: 404, title: "fiber peer not found", detail: "No peer found by given peer id", href: "" end end class FiberChannelNotFoundError < Error def initialize - super code: 2012, status: 404, title: "fiber channel Not Found", detail: "No channel found by given channel_id", href: "" + super code: 2012, status: 404, title: "fiber channel not found", detail: "No channel found by given channel_id", href: "" + end + end + + class FiberGraphNodeNotFoundError < Error + def initialize + super code: 2013, status: 404, title: "fiber graph node not found", detail: "No graph node found by given node id", href: "" end end end diff --git a/app/models/fiber_channel.rb b/app/models/fiber_channel.rb index 6272408b2..6c6d98c3c 100644 --- a/app/models/fiber_channel.rb +++ b/app/models/fiber_channel.rb @@ -1,6 +1,5 @@ class FiberChannel < ApplicationRecord belongs_to :fiber_peer - # has_many :fiber_transactions def local_peer fiber_peer || FiberPeer.new diff --git a/app/models/fiber_graph_channel.rb b/app/models/fiber_graph_channel.rb new file mode 100644 index 000000000..494f793df --- /dev/null +++ b/app/models/fiber_graph_channel.rb @@ -0,0 +1,30 @@ +class FiberGraphChannel < ApplicationRecord + MAX_PAGINATES_PER = 100 + DEFAULT_PAGINATES_PER = 10 + paginates_per DEFAULT_PAGINATES_PER + max_paginates_per MAX_PAGINATES_PER +end + +# == Schema Information +# +# Table name: fiber_graph_channels +# +# id :bigint not null, primary key +# channel_outpoint :string +# funding_tx_block_number :bigint +# funding_tx_index :integer +# node1 :string +# node2 :string +# last_updated_timestamp :bigint +# created_timestamp :bigint +# node1_to_node2_fee_rate :decimal(30, ) default(0) +# node2_to_node1_fee_rate :decimal(30, ) default(0) +# capacity :decimal(64, 2) default(0.0) +# chain_hash :string +# created_at :datetime not null +# updated_at :datetime not null +# +# Indexes +# +# index_fiber_graph_channels_on_channel_outpoint (channel_outpoint) UNIQUE +# diff --git a/app/models/fiber_graph_node.rb b/app/models/fiber_graph_node.rb new file mode 100644 index 000000000..0f9843f28 --- /dev/null +++ b/app/models/fiber_graph_node.rb @@ -0,0 +1,25 @@ +class FiberGraphNode < ApplicationRecord + MAX_PAGINATES_PER = 100 + DEFAULT_PAGINATES_PER = 10 + paginates_per DEFAULT_PAGINATES_PER + max_paginates_per MAX_PAGINATES_PER +end + +# == Schema Information +# +# Table name: fiber_graph_nodes +# +# id :bigint not null, primary key +# alias :string +# node_id :string +# addresses :string default([]), is an Array +# timestamp :bigint +# chain_hash :string +# auto_accept_min_ckb_funding_amount :decimal(30, ) +# created_at :datetime not null +# updated_at :datetime not null +# +# Indexes +# +# index_fiber_graph_nodes_on_node_id (node_id) UNIQUE +# diff --git a/app/models/fiber_peer.rb b/app/models/fiber_peer.rb index 9edd9bbde..616e3c1d0 100644 --- a/app/models/fiber_peer.rb +++ b/app/models/fiber_peer.rb @@ -5,7 +5,6 @@ class FiberPeer < ApplicationRecord max_paginates_per MAX_PAGINATES_PER has_many :fiber_channels, dependent: :destroy - # has_many :fiber_transactions validates :peer_id, presence: true, uniqueness: true @@ -30,4 +29,10 @@ def channels_count # last_channel_updated_at :datetime # created_at :datetime not null # updated_at :datetime not null +# node_id :string +# chain_hash :string +# +# Indexes +# +# index_fiber_peers_on_peer_id (peer_id) UNIQUE # diff --git a/app/services/fiber_coordinator.rb b/app/services/fiber_coordinator.rb index e36e19ba4..a8f143a9a 100644 --- a/app/services/fiber_coordinator.rb +++ b/app/services/fiber_coordinator.rb @@ -1,6 +1,6 @@ class FiberCoordinator include Singleton - METHOD_NAMES = %w(list_channels).freeze + METHOD_NAMES = %w(graph_nodes graph_channels list_channels).freeze def initialize @id = 0 diff --git a/app/views/api/v2/fiber/graph_channels/index.jbuilder b/app/views/api/v2/fiber/graph_channels/index.jbuilder new file mode 100644 index 000000000..d886aa041 --- /dev/null +++ b/app/views/api/v2/fiber/graph_channels/index.jbuilder @@ -0,0 +1,16 @@ +json.data do + json.fiber_graph_channels @channels do |channel| + json.(channel, :channel_outpoint, :node1, :node2, :chain_hash) + json.funding_tx_block_number channel.funding_tx_block_number.to_s + json.funding_tx_index channel.funding_tx_index.to_s + json.last_updated_timestamp channel.last_updated_timestamp.to_s + json.created_timestamp channel.created_timestamp.to_s + json.node1_to_node2_fee_rate channel.node1_to_node2_fee_rate.to_s + json.node2_to_node1_fee_rate channel.node2_to_node1_fee_rate.to_s + json.capacity channel.capacity.to_s + end +end +json.meta do + json.total @channels.total_count + json.page_size @page_size.to_i +end diff --git a/app/views/api/v2/fiber/graph_nodes/index.jbuilder b/app/views/api/v2/fiber/graph_nodes/index.jbuilder new file mode 100644 index 000000000..b715ab7b7 --- /dev/null +++ b/app/views/api/v2/fiber/graph_nodes/index.jbuilder @@ -0,0 +1,11 @@ +json.data do + json.fiber_graph_nodes @nodes do |node| + json.(node, :alias, :node_id, :addresses, :timestamp, :chain_hash) + json.timestamp node.timestamp.to_s + json.auto_accept_min_ckb_funding_amount node.auto_accept_min_ckb_funding_amount.to_s + end +end +json.meta do + json.total @nodes.total_count + json.page_size @page_size.to_i +end diff --git a/app/views/api/v2/fiber/graph_nodes/show.jbuilder b/app/views/api/v2/fiber/graph_nodes/show.jbuilder new file mode 100644 index 000000000..99a078639 --- /dev/null +++ b/app/views/api/v2/fiber/graph_nodes/show.jbuilder @@ -0,0 +1,16 @@ +json.data do + json.(@node, :alias, :node_id, :addresses, :timestamp, :chain_hash) + json.timestamp @node.timestamp.to_s + json.auto_accept_min_ckb_funding_amount @node.auto_accept_min_ckb_funding_amount.to_s + + json.fiber_graph_channels @graph_channels do |channel| + json.(channel, :channel_outpoint, :node1, :node2, :chain_hash) + json.funding_tx_block_number channel.funding_tx_block_number.to_s + json.funding_tx_index channel.funding_tx_index.to_s + json.last_updated_timestamp channel.last_updated_timestamp.to_s + json.created_timestamp channel.created_timestamp.to_s + json.node1_to_node2_fee_rate channel.node1_to_node2_fee_rate.to_s + json.node2_to_node1_fee_rate channel.node2_to_node1_fee_rate.to_s + json.capacity channel.capacity.to_s + end +end diff --git a/app/workers/fiber_detect_worker.rb b/app/workers/fiber_detect_worker.rb index ee8267748..6fef6759b 100644 --- a/app/workers/fiber_detect_worker.rb +++ b/app/workers/fiber_detect_worker.rb @@ -2,7 +2,7 @@ class FiberDetectWorker include Sidekiq::Worker sidekiq_options queue: "fiber" - def perform(peer_id) + def perform(peer_id = nil) fiber_peers = peer_id.present? ? FiberPeer.where(peer_id:) : FiberPeer.all fiber_peers.each { sync_with_fiber_channels(_1) } end diff --git a/app/workers/fiber_graph_detect_worker.rb b/app/workers/fiber_graph_detect_worker.rb new file mode 100644 index 000000000..f95e81032 --- /dev/null +++ b/app/workers/fiber_graph_detect_worker.rb @@ -0,0 +1,78 @@ +class FiberGraphDetectWorker + include Sidekiq::Worker + sidekiq_options queue: "fiber" + + def perform + ["nodes", "channels"].each { fetch_graph_infos(_1) } + end + + private + + def fetch_graph_infos(data_type) + cursor = nil + + loop do + break if cursor == "0x" + + next_cursor = send("fetch_#{data_type}", cursor) + break if next_cursor.nil? || next_cursor == cursor + + cursor = next_cursor + end + end + + def fetch_nodes(last_cursor) + return if ENV["FIBER_NODE_URL"].blank? + + data = rpc.graph_nodes(ENV["FIBER_NODE_URL"], { limit: 1, after: last_cursor }) + node_attributes = data["result"]["nodes"].map do |node| + { + alias: node["alias"], + node_id: node["node_id"], + addresses: node["addresses"], + timestamp: node["timestamp"].to_i(16), + chain_hash: node["chain_hash"], + auto_accept_min_ckb_funding_amount: node["auto_accept_min_ckb_funding_amount"], + } + end + + FiberGraphNode.upsert_all(node_attributes, unique_by: %i[node_id]) if node_attributes.any? + + data["result"]["last_cursor"] + rescue StandardError => e + Rails.logger.error("Error fetching nodes: #{e.message}") + nil + end + + def fetch_channels(last_cursor) + return if ENV["FIBER_NODE_URL"].blank? + + data = rpc.graph_channels(ENV["FIBER_NODE_URL"], { limit: 1, after: last_cursor }) + channel_attributes = data["result"]["channels"].map do |channel| + { + channel_outpoint: channel["channel_outpoint"], + funding_tx_block_number: channel["funding_tx_block_number"].to_i(16), + funding_tx_index: channel["funding_tx_index"].to_i(16), + node1: channel["node1"], + node2: channel["node2"], + last_updated_timestamp: channel["last_updated_timestamp"].to_i(16), + created_timestamp: channel["created_timestamp"], + node1_to_node2_fee_rate: channel["node1_to_node2_fee_rate"].to_i(16), + node2_to_node1_fee_rate: channel["node2_to_node1_fee_rate"].to_i(16), + capacity: channel["capacity"].to_i(16), + chain_hash: channel["chain_hash"], + } + end + + FiberGraphChannel.upsert_all(channel_attributes, unique_by: %i[channel_outpoint]) if channel_attributes.any? + + data["result"]["last_cursor"] + rescue StandardError => e + Rails.logger.error("Error fetching channels: #{e.message}") + nil + end + + def rpc + @rpc ||= FiberCoordinator.instance + end +end diff --git a/config/routes/v2.rb b/config/routes/v2.rb index b0cbf8fcc..6a74404f9 100644 --- a/config/routes/v2.rb +++ b/config/routes/v2.rb @@ -103,6 +103,8 @@ namespace :fiber do resources :peers, param: :peer_id, only: %i[index show create] resources :channels, param: :channel_id, only: :show + resources :graph_nodes, param: :node_id, only: %i[index show] + resources :graph_channels, only: :index end end end diff --git a/db/migrate/20241012014906_create_fiber_graph_infos.rb b/db/migrate/20241012014906_create_fiber_graph_infos.rb new file mode 100644 index 000000000..39637c7d0 --- /dev/null +++ b/db/migrate/20241012014906_create_fiber_graph_infos.rb @@ -0,0 +1,33 @@ +class CreateFiberGraphInfos < ActiveRecord::Migration[7.0] + def change + create_table :fiber_graph_nodes do |t| + t.string :alias + t.string :node_id + t.string :addresses, array: true, default: [], using: "(string_to_array(addresses, ','))" + t.bigint :timestamp + t.string :chain_hash + t.decimal :auto_accept_min_ckb_funding_amount, precision: 30 + + t.timestamps + end + + create_table :fiber_graph_channels do |t| + t.string :channel_outpoint + t.bigint :funding_tx_block_number + t.integer :funding_tx_index + t.string :node1 + t.string :node2 + t.bigint :last_updated_timestamp + t.bigint :created_timestamp + t.decimal :node1_to_node2_fee_rate, precision: 30, default: 0.0 + t.decimal :node2_to_node1_fee_rate, precision: 30, default: 0.0 + t.decimal :capacity, precision: 64, scale: 2, default: 0.0 + t.string :chain_hash + + t.timestamps + end + + add_index :fiber_graph_nodes, :node_id, unique: true + add_index :fiber_graph_channels, :channel_outpoint, unique: true + end +end diff --git a/db/structure.sql b/db/structure.sql index 8fb54337e..4ce74828a 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -1744,6 +1744,83 @@ CREATE SEQUENCE public.fiber_channels_id_seq ALTER SEQUENCE public.fiber_channels_id_seq OWNED BY public.fiber_channels.id; +-- +-- Name: fiber_graph_channels; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.fiber_graph_channels ( + id bigint NOT NULL, + channel_outpoint character varying, + funding_tx_block_number bigint, + funding_tx_index integer, + node1 character varying, + node2 character varying, + last_updated_timestamp bigint, + created_timestamp bigint, + node1_to_node2_fee_rate numeric(30,0) DEFAULT 0.0, + node2_to_node1_fee_rate numeric(30,0) DEFAULT 0.0, + capacity numeric(64,2) DEFAULT 0.0, + chain_hash character varying, + created_at timestamp(6) without time zone NOT NULL, + updated_at timestamp(6) without time zone NOT NULL +); + + +-- +-- Name: fiber_graph_channels_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.fiber_graph_channels_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: fiber_graph_channels_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.fiber_graph_channels_id_seq OWNED BY public.fiber_graph_channels.id; + + +-- +-- Name: fiber_graph_nodes; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.fiber_graph_nodes ( + id bigint NOT NULL, + alias character varying, + node_id character varying, + addresses character varying[] DEFAULT '{}'::character varying[], + "timestamp" bigint, + chain_hash character varying, + auto_accept_min_ckb_funding_amount numeric(30,0), + created_at timestamp(6) without time zone NOT NULL, + updated_at timestamp(6) without time zone NOT NULL +); + + +-- +-- Name: fiber_graph_nodes_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.fiber_graph_nodes_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: fiber_graph_nodes_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.fiber_graph_nodes_id_seq OWNED BY public.fiber_graph_nodes.id; + + -- -- Name: fiber_peers; Type: TABLE; Schema: public; Owner: - -- @@ -1756,7 +1833,9 @@ CREATE TABLE public.fiber_peers ( first_channel_opened_at timestamp(6) without time zone, last_channel_updated_at timestamp(6) without time zone, created_at timestamp(6) without time zone NOT NULL, - updated_at timestamp(6) without time zone NOT NULL + updated_at timestamp(6) without time zone NOT NULL, + node_id character varying, + chain_hash character varying ); @@ -3199,6 +3278,20 @@ ALTER TABLE ONLY public.epoch_statistics ALTER COLUMN id SET DEFAULT nextval('pu ALTER TABLE ONLY public.fiber_channels ALTER COLUMN id SET DEFAULT nextval('public.fiber_channels_id_seq'::regclass); +-- +-- Name: fiber_graph_channels id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.fiber_graph_channels ALTER COLUMN id SET DEFAULT nextval('public.fiber_graph_channels_id_seq'::regclass); + + +-- +-- Name: fiber_graph_nodes id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.fiber_graph_nodes ALTER COLUMN id SET DEFAULT nextval('public.fiber_graph_nodes_id_seq'::regclass); + + -- -- Name: fiber_peers id; Type: DEFAULT; Schema: public; Owner: - -- @@ -3760,6 +3853,22 @@ ALTER TABLE ONLY public.fiber_channels ADD CONSTRAINT fiber_channels_pkey PRIMARY KEY (id); +-- +-- Name: fiber_graph_channels fiber_graph_channels_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.fiber_graph_channels + ADD CONSTRAINT fiber_graph_channels_pkey PRIMARY KEY (id); + + +-- +-- Name: fiber_graph_nodes fiber_graph_nodes_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.fiber_graph_nodes + ADD CONSTRAINT fiber_graph_nodes_pkey PRIMARY KEY (id); + + -- -- Name: fiber_peers fiber_peers_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -4943,6 +5052,27 @@ CREATE INDEX index_fiber_channels_on_fiber_peer_id ON public.fiber_channels USIN CREATE UNIQUE INDEX index_fiber_channels_on_peer_id_and_channel_id ON public.fiber_channels USING btree (peer_id, channel_id); +-- +-- Name: index_fiber_graph_channels_on_channel_outpoint; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX index_fiber_graph_channels_on_channel_outpoint ON public.fiber_graph_channels USING btree (channel_outpoint); + + +-- +-- Name: index_fiber_graph_nodes_on_node_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX index_fiber_graph_nodes_on_node_id ON public.fiber_graph_nodes USING btree (node_id); + + +-- +-- Name: index_fiber_peers_on_peer_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX index_fiber_peers_on_peer_id ON public.fiber_peers USING btree (peer_id); + + -- -- Name: index_forked_events_on_status; Type: INDEX; Schema: public; Owner: - -- @@ -6165,6 +6295,8 @@ INSERT INTO "schema_migrations" (version) VALUES ('20240918024415'), ('20240918033146'), ('20240920094807'), -('20240924065539'); +('20240924065539'), +('20241009081935'), +('20241012014906'); diff --git a/lib/scheduler.rb b/lib/scheduler.rb index da0981a17..d31b5dcdf 100644 --- a/lib/scheduler.rb +++ b/lib/scheduler.rb @@ -129,7 +129,8 @@ def call_worker(clz) end s.every "5m", overlap: false do - call_worker FiberDetectWorker + # call_worker FiberDetectWorker + call_worker FiberGraphDetectWorker end s.join diff --git a/test/factories/fiber_graph_channels.rb b/test/factories/fiber_graph_channels.rb new file mode 100644 index 000000000..f63ba3d55 --- /dev/null +++ b/test/factories/fiber_graph_channels.rb @@ -0,0 +1,5 @@ +FactoryBot.define do + factory :fiber_graph_channel do + + end +end diff --git a/test/factories/fiber_transactions.rb b/test/factories/fiber_graph_nodes.rb similarity index 52% rename from test/factories/fiber_transactions.rb rename to test/factories/fiber_graph_nodes.rb index a0e85493d..a387eeb93 100644 --- a/test/factories/fiber_transactions.rb +++ b/test/factories/fiber_graph_nodes.rb @@ -1,5 +1,5 @@ FactoryBot.define do - factory :fiber_transaction do + factory :fiber_graph_node do end end diff --git a/test/models/fiber_graph_channel_test.rb b/test/models/fiber_graph_channel_test.rb new file mode 100644 index 000000000..d9674a776 --- /dev/null +++ b/test/models/fiber_graph_channel_test.rb @@ -0,0 +1,7 @@ +require "test_helper" + +class FiberGraphChannelTest < ActiveSupport::TestCase + # test "the truth" do + # assert true + # end +end diff --git a/test/models/fiber_transaction_test.rb b/test/models/fiber_graph_node_test.rb similarity index 59% rename from test/models/fiber_transaction_test.rb rename to test/models/fiber_graph_node_test.rb index 01f7bde69..ef430b005 100644 --- a/test/models/fiber_transaction_test.rb +++ b/test/models/fiber_graph_node_test.rb @@ -1,6 +1,6 @@ require "test_helper" -class FiberTransactionTest < ActiveSupport::TestCase +class FiberGraphNodeTest < ActiveSupport::TestCase # test "the truth" do # assert true # end From b9de15ab3f18fb92ccda2bff76f191bcd60d2218 Mon Sep 17 00:00:00 2001 From: Rabbit Date: Tue, 22 Oct 2024 15:09:19 +0800 Subject: [PATCH 09/15] fix: set default page_size for fiber request --- app/workers/fiber_graph_detect_worker.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/workers/fiber_graph_detect_worker.rb b/app/workers/fiber_graph_detect_worker.rb index f95e81032..383cd0645 100644 --- a/app/workers/fiber_graph_detect_worker.rb +++ b/app/workers/fiber_graph_detect_worker.rb @@ -24,7 +24,7 @@ def fetch_graph_infos(data_type) def fetch_nodes(last_cursor) return if ENV["FIBER_NODE_URL"].blank? - data = rpc.graph_nodes(ENV["FIBER_NODE_URL"], { limit: 1, after: last_cursor }) + data = rpc.graph_nodes(ENV["FIBER_NODE_URL"], { limit: "0x100", after: last_cursor }) node_attributes = data["result"]["nodes"].map do |node| { alias: node["alias"], @@ -47,7 +47,7 @@ def fetch_nodes(last_cursor) def fetch_channels(last_cursor) return if ENV["FIBER_NODE_URL"].blank? - data = rpc.graph_channels(ENV["FIBER_NODE_URL"], { limit: 1, after: last_cursor }) + data = rpc.graph_channels(ENV["FIBER_NODE_URL"], { limit: "0x100", after: last_cursor }) channel_attributes = data["result"]["channels"].map do |channel| { channel_outpoint: channel["channel_outpoint"], From a5081378e6433a7da4c59653afce256969c70cf9 Mon Sep 17 00:00:00 2001 From: Rabbit Date: Wed, 23 Oct 2024 15:55:20 +0800 Subject: [PATCH 10/15] feat: sync graph node udt config info --- app/models/fiber_graph_channel.rb | 11 ++++ app/models/fiber_graph_node.rb | 19 ++++++ app/models/fiber_udt_cfg_info.rb | 24 +++++++ .../v2/fiber/graph_channels/index.jbuilder | 3 +- .../api/v2/fiber/graph_nodes/index.jbuilder | 4 +- .../api/v2/fiber/graph_nodes/show.jbuilder | 6 +- app/workers/fiber_graph_detect_worker.rb | 47 ++++++++++---- ...241023055256_create_fiber_udt_cfg_infos.rb | 13 ++++ ...3536_add_udt_id_to_fiber_graph_channels.rb | 5 ++ db/structure.sql | 62 ++++++++++++++++++- test/factories/fiber_udt_cfg_infos.rb | 5 ++ test/models/fiber_udt_cfg_info_test.rb | 7 +++ 12 files changed, 189 insertions(+), 17 deletions(-) create mode 100644 app/models/fiber_udt_cfg_info.rb create mode 100644 db/migrate/20241023055256_create_fiber_udt_cfg_infos.rb create mode 100644 db/migrate/20241023063536_add_udt_id_to_fiber_graph_channels.rb create mode 100644 test/factories/fiber_udt_cfg_infos.rb create mode 100644 test/models/fiber_udt_cfg_info_test.rb diff --git a/app/models/fiber_graph_channel.rb b/app/models/fiber_graph_channel.rb index 494f793df..c2db28dcd 100644 --- a/app/models/fiber_graph_channel.rb +++ b/app/models/fiber_graph_channel.rb @@ -3,6 +3,16 @@ class FiberGraphChannel < ApplicationRecord DEFAULT_PAGINATES_PER = 10 paginates_per DEFAULT_PAGINATES_PER max_paginates_per MAX_PAGINATES_PER + + belongs_to :udt, optional: true + + def open_transaction_hash + channel_outpoint[0..65] + end + + def udt_info + udt&.as_json(only: %i[full_name symbol decimal icon_file]) + end end # == Schema Information @@ -23,6 +33,7 @@ class FiberGraphChannel < ApplicationRecord # chain_hash :string # created_at :datetime not null # updated_at :datetime not null +# udt_id :bigint # # Indexes # diff --git a/app/models/fiber_graph_node.rb b/app/models/fiber_graph_node.rb index 0f9843f28..4380c3639 100644 --- a/app/models/fiber_graph_node.rb +++ b/app/models/fiber_graph_node.rb @@ -3,6 +3,25 @@ class FiberGraphNode < ApplicationRecord DEFAULT_PAGINATES_PER = 10 paginates_per DEFAULT_PAGINATES_PER max_paginates_per MAX_PAGINATES_PER + + has_many :fiber_udt_cfg_infos, dependent: :delete_all + + def channel_links + FiberGraphChannel.where(node1: node_id).or(FiberGraphChannel.where(node2: node_id)) + end + + def udt_cfg_infos + fiber_udt_cfg_infos.map(&:udt_info) + end + + def total_capacity + channel_links.sum(&:capacity) + end + + def connected_node_ids + node_ids = channel_links.pluck(:node1, :node2).flatten + node_ids.uniq - [node_id] + end end # == Schema Information diff --git a/app/models/fiber_udt_cfg_info.rb b/app/models/fiber_udt_cfg_info.rb new file mode 100644 index 000000000..e1d836c53 --- /dev/null +++ b/app/models/fiber_udt_cfg_info.rb @@ -0,0 +1,24 @@ +class FiberUdtCfgInfo < ApplicationRecord + belongs_to :fiber_graph_node + belongs_to :udt + + def udt_info + udt.as_json(only: %i[full_name symbol decimal icon_file]) + end +end + +# == Schema Information +# +# Table name: fiber_udt_cfg_infos +# +# id :bigint not null, primary key +# fiber_graph_node_id :bigint +# udt_id :bigint +# auto_accept_amount :decimal(64, 2) default(0.0) +# created_at :datetime not null +# updated_at :datetime not null +# +# Indexes +# +# index_fiber_udt_cfg_infos_on_fiber_graph_node_id_and_udt_id (fiber_graph_node_id,udt_id) UNIQUE +# diff --git a/app/views/api/v2/fiber/graph_channels/index.jbuilder b/app/views/api/v2/fiber/graph_channels/index.jbuilder index d886aa041..f38f4e022 100644 --- a/app/views/api/v2/fiber/graph_channels/index.jbuilder +++ b/app/views/api/v2/fiber/graph_channels/index.jbuilder @@ -1,6 +1,6 @@ json.data do json.fiber_graph_channels @channels do |channel| - json.(channel, :channel_outpoint, :node1, :node2, :chain_hash) + json.(channel, :channel_outpoint, :node1, :node2, :chain_hash, :open_transaction_hash) json.funding_tx_block_number channel.funding_tx_block_number.to_s json.funding_tx_index channel.funding_tx_index.to_s json.last_updated_timestamp channel.last_updated_timestamp.to_s @@ -8,6 +8,7 @@ json.data do json.node1_to_node2_fee_rate channel.node1_to_node2_fee_rate.to_s json.node2_to_node1_fee_rate channel.node2_to_node1_fee_rate.to_s json.capacity channel.capacity.to_s + json.udt_cfg_info channel.udt_info end end json.meta do diff --git a/app/views/api/v2/fiber/graph_nodes/index.jbuilder b/app/views/api/v2/fiber/graph_nodes/index.jbuilder index b715ab7b7..1a4936d7e 100644 --- a/app/views/api/v2/fiber/graph_nodes/index.jbuilder +++ b/app/views/api/v2/fiber/graph_nodes/index.jbuilder @@ -1,8 +1,10 @@ json.data do json.fiber_graph_nodes @nodes do |node| - json.(node, :alias, :node_id, :addresses, :timestamp, :chain_hash) + json.(node, :alias, :node_id, :addresses, :timestamp, :chain_hash, :connected_node_ids) json.timestamp node.timestamp.to_s json.auto_accept_min_ckb_funding_amount node.auto_accept_min_ckb_funding_amount.to_s + json.total_capacity node.total_capacity.to_s + json.udt_cfg_infos node.udt_cfg_infos end end json.meta do diff --git a/app/views/api/v2/fiber/graph_nodes/show.jbuilder b/app/views/api/v2/fiber/graph_nodes/show.jbuilder index 99a078639..5b7db4ced 100644 --- a/app/views/api/v2/fiber/graph_nodes/show.jbuilder +++ b/app/views/api/v2/fiber/graph_nodes/show.jbuilder @@ -1,10 +1,12 @@ json.data do - json.(@node, :alias, :node_id, :addresses, :timestamp, :chain_hash) + json.(@node, :alias, :node_id, :addresses, :timestamp, :chain_hash, :connected_node_ids) json.timestamp @node.timestamp.to_s json.auto_accept_min_ckb_funding_amount @node.auto_accept_min_ckb_funding_amount.to_s + json.total_capacity @node.total_capacity.to_s + json.udt_cfg_infos @node.udt_cfg_infos json.fiber_graph_channels @graph_channels do |channel| - json.(channel, :channel_outpoint, :node1, :node2, :chain_hash) + json.(channel, :channel_outpoint, :node1, :node2, :chain_hash, :open_transaction_hash) json.funding_tx_block_number channel.funding_tx_block_number.to_s json.funding_tx_index channel.funding_tx_index.to_s json.last_updated_timestamp channel.last_updated_timestamp.to_s diff --git a/app/workers/fiber_graph_detect_worker.rb b/app/workers/fiber_graph_detect_worker.rb index 383cd0645..64c86282a 100644 --- a/app/workers/fiber_graph_detect_worker.rb +++ b/app/workers/fiber_graph_detect_worker.rb @@ -25,18 +25,38 @@ def fetch_nodes(last_cursor) return if ENV["FIBER_NODE_URL"].blank? data = rpc.graph_nodes(ENV["FIBER_NODE_URL"], { limit: "0x100", after: last_cursor }) - node_attributes = data["result"]["nodes"].map do |node| - { - alias: node["alias"], - node_id: node["node_id"], - addresses: node["addresses"], - timestamp: node["timestamp"].to_i(16), - chain_hash: node["chain_hash"], - auto_accept_min_ckb_funding_amount: node["auto_accept_min_ckb_funding_amount"], - } - end + ApplicationRecord.transaction do + data["result"]["nodes"].each do |node| + node_attributes = { + alias: node["alias"], + node_id: node["node_id"], + addresses: node["addresses"], + timestamp: node["timestamp"].to_i(16), + chain_hash: node["chain_hash"], + auto_accept_min_ckb_funding_amount: node["auto_accept_min_ckb_funding_amount"], + } + + fiber_graph_node = FiberGraphNode.upsert(node_attributes, unique_by: %i[node_id], returning: %i[id]) - FiberGraphNode.upsert_all(node_attributes, unique_by: %i[node_id]) if node_attributes.any? + cfg_info_attributes = [] + node["udt_cfg_infos"].each do |info| + udt = Udt.find_by(info["script"].symbolize_keys) + + if udt + cfg_info_attributes << { + fiber_graph_node_id: fiber_graph_node[0]["id"], + udt_id: udt.id, + auto_accept_amount: info["auto_accept_amount"].to_i(16), + } + end + end + + if cfg_info_attributes.present? + puts + FiberUdtCfgInfo.upsert_all(cfg_info_attributes, unique_by: %i[fiber_graph_node_id udt_id]) + end + end + end data["result"]["last_cursor"] rescue StandardError => e @@ -49,6 +69,10 @@ def fetch_channels(last_cursor) data = rpc.graph_channels(ENV["FIBER_NODE_URL"], { limit: "0x100", after: last_cursor }) channel_attributes = data["result"]["channels"].map do |channel| + if (udt_type_script = channel["udt_type_script"]).present? + udt = Udt.find_by(type_hash: udt_type_script) + end + { channel_outpoint: channel["channel_outpoint"], funding_tx_block_number: channel["funding_tx_block_number"].to_i(16), @@ -61,6 +85,7 @@ def fetch_channels(last_cursor) node2_to_node1_fee_rate: channel["node2_to_node1_fee_rate"].to_i(16), capacity: channel["capacity"].to_i(16), chain_hash: channel["chain_hash"], + udt_id: udt&.id, } end diff --git a/db/migrate/20241023055256_create_fiber_udt_cfg_infos.rb b/db/migrate/20241023055256_create_fiber_udt_cfg_infos.rb new file mode 100644 index 000000000..54faa718a --- /dev/null +++ b/db/migrate/20241023055256_create_fiber_udt_cfg_infos.rb @@ -0,0 +1,13 @@ +class CreateFiberUdtCfgInfos < ActiveRecord::Migration[7.0] + def change + create_table :fiber_udt_cfg_infos do |t| + t.bigint :fiber_graph_node_id + t.bigint :udt_id + t.decimal :auto_accept_amount, precision: 64, scale: 2, default: 0.0 + + t.timestamps + end + + add_index :fiber_udt_cfg_infos, %i[fiber_graph_node_id udt_id], unique: true + end +end diff --git a/db/migrate/20241023063536_add_udt_id_to_fiber_graph_channels.rb b/db/migrate/20241023063536_add_udt_id_to_fiber_graph_channels.rb new file mode 100644 index 000000000..20c001b16 --- /dev/null +++ b/db/migrate/20241023063536_add_udt_id_to_fiber_graph_channels.rb @@ -0,0 +1,5 @@ +class AddUdtIdToFiberGraphChannels < ActiveRecord::Migration[7.0] + def change + add_column :fiber_graph_channels, :udt_id, :bigint + end +end diff --git a/db/structure.sql b/db/structure.sql index 4ce74828a..3d4aa23c1 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -1762,7 +1762,8 @@ CREATE TABLE public.fiber_graph_channels ( capacity numeric(64,2) DEFAULT 0.0, chain_hash character varying, created_at timestamp(6) without time zone NOT NULL, - updated_at timestamp(6) without time zone NOT NULL + updated_at timestamp(6) without time zone NOT NULL, + udt_id bigint ); @@ -1891,6 +1892,39 @@ CREATE SEQUENCE public.fiber_transactions_id_seq ALTER SEQUENCE public.fiber_transactions_id_seq OWNED BY public.fiber_transactions.id; +-- +-- Name: fiber_udt_cfg_infos; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.fiber_udt_cfg_infos ( + id bigint NOT NULL, + fiber_graph_node_id bigint, + udt_id bigint, + auto_accept_amount numeric(64,2) DEFAULT 0.0, + created_at timestamp(6) without time zone NOT NULL, + updated_at timestamp(6) without time zone NOT NULL +); + + +-- +-- Name: fiber_udt_cfg_infos_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.fiber_udt_cfg_infos_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: fiber_udt_cfg_infos_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.fiber_udt_cfg_infos_id_seq OWNED BY public.fiber_udt_cfg_infos.id; + + -- -- Name: forked_blocks; Type: TABLE; Schema: public; Owner: - -- @@ -3306,6 +3340,13 @@ ALTER TABLE ONLY public.fiber_peers ALTER COLUMN id SET DEFAULT nextval('public. ALTER TABLE ONLY public.fiber_transactions ALTER COLUMN id SET DEFAULT nextval('public.fiber_transactions_id_seq'::regclass); +-- +-- Name: fiber_udt_cfg_infos id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.fiber_udt_cfg_infos ALTER COLUMN id SET DEFAULT nextval('public.fiber_udt_cfg_infos_id_seq'::regclass); + + -- -- Name: forked_blocks id; Type: DEFAULT; Schema: public; Owner: - -- @@ -3885,6 +3926,14 @@ ALTER TABLE ONLY public.fiber_transactions ADD CONSTRAINT fiber_transactions_pkey PRIMARY KEY (id); +-- +-- Name: fiber_udt_cfg_infos fiber_udt_cfg_infos_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.fiber_udt_cfg_infos + ADD CONSTRAINT fiber_udt_cfg_infos_pkey PRIMARY KEY (id); + + -- -- Name: forked_blocks forked_blocks_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -5073,6 +5122,13 @@ CREATE UNIQUE INDEX index_fiber_graph_nodes_on_node_id ON public.fiber_graph_nod CREATE UNIQUE INDEX index_fiber_peers_on_peer_id ON public.fiber_peers USING btree (peer_id); +-- +-- Name: index_fiber_udt_cfg_infos_on_fiber_graph_node_id_and_udt_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX index_fiber_udt_cfg_infos_on_fiber_graph_node_id_and_udt_id ON public.fiber_udt_cfg_infos USING btree (fiber_graph_node_id, udt_id); + + -- -- Name: index_forked_events_on_status; Type: INDEX; Schema: public; Owner: - -- @@ -6297,6 +6353,8 @@ INSERT INTO "schema_migrations" (version) VALUES ('20240920094807'), ('20240924065539'), ('20241009081935'), -('20241012014906'); +('20241012014906'), +('20241023055256'), +('20241023063536'); diff --git a/test/factories/fiber_udt_cfg_infos.rb b/test/factories/fiber_udt_cfg_infos.rb new file mode 100644 index 000000000..f9cbd8b2a --- /dev/null +++ b/test/factories/fiber_udt_cfg_infos.rb @@ -0,0 +1,5 @@ +FactoryBot.define do + factory :fiber_udt_cfg_info do + + end +end diff --git a/test/models/fiber_udt_cfg_info_test.rb b/test/models/fiber_udt_cfg_info_test.rb new file mode 100644 index 000000000..80fdf45d9 --- /dev/null +++ b/test/models/fiber_udt_cfg_info_test.rb @@ -0,0 +1,7 @@ +require "test_helper" + +class FiberUdtCfgInfoTest < ActiveSupport::TestCase + # test "the truth" do + # assert true + # end +end From 0b75eb44849edaea10384cb418b0715a9eb04628 Mon Sep 17 00:00:00 2001 From: Rabbit Date: Thu, 24 Oct 2024 09:37:55 +0800 Subject: [PATCH 11/15] chore: fiber udt cfg info add type_script --- app/models/fiber_udt_cfg_info.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/fiber_udt_cfg_info.rb b/app/models/fiber_udt_cfg_info.rb index e1d836c53..fb5d9b8d6 100644 --- a/app/models/fiber_udt_cfg_info.rb +++ b/app/models/fiber_udt_cfg_info.rb @@ -3,7 +3,7 @@ class FiberUdtCfgInfo < ApplicationRecord belongs_to :udt def udt_info - udt.as_json(only: %i[full_name symbol decimal icon_file]) + udt.as_json(only: %i[full_name symbol decimal icon_file args code_hash hash_type]).merge(auto_accept_amount:) end end From 12f1a79d543d27b74294d4f261d5f6f76e02aae5 Mon Sep 17 00:00:00 2001 From: Rabbit Date: Wed, 30 Oct 2024 16:56:12 +0800 Subject: [PATCH 12/15] refactor: fiber graph detect worker --- .../api/v2/fiber/graph_nodes_controller.rb | 4 + app/models/fiber_graph_channel.rb | 24 +++- app/models/fiber_graph_node.rb | 4 + .../v2/fiber/graph_channels/index.jbuilder | 2 +- .../api/v2/fiber/graph_nodes/index.jbuilder | 3 +- .../api/v2/fiber/graph_nodes/show.jbuilder | 2 +- app/workers/fiber_graph_detect_worker.rb | 129 +++++++++--------- config/settings.mainnet.yml | 3 + config/settings.testnet.yml | 3 + ..._transaction_id_to_fiber_graph_channels.rb | 8 ++ db/structure.sql | 7 +- lib/scheduler.rb | 1 - 12 files changed, 121 insertions(+), 69 deletions(-) create mode 100644 db/migrate/20241030023309_add_transaction_id_to_fiber_graph_channels.rb diff --git a/app/controllers/api/v2/fiber/graph_nodes_controller.rb b/app/controllers/api/v2/fiber/graph_nodes_controller.rb index 60fd220f3..ec8aee334 100644 --- a/app/controllers/api/v2/fiber/graph_nodes_controller.rb +++ b/app/controllers/api/v2/fiber/graph_nodes_controller.rb @@ -15,6 +15,10 @@ def show @graph_channels = FiberGraphChannel.where(node1: params[:node_id]).or( FiberGraphChannel.where(node2: params[:node_id]), ) + + if params[:status] == "closed" + @graph_channels = @graph_channels.where.not(closed_transaction_id: nil) + end end end end diff --git a/app/models/fiber_graph_channel.rb b/app/models/fiber_graph_channel.rb index c2db28dcd..d83c16d88 100644 --- a/app/models/fiber_graph_channel.rb +++ b/app/models/fiber_graph_channel.rb @@ -5,14 +5,32 @@ class FiberGraphChannel < ApplicationRecord max_paginates_per MAX_PAGINATES_PER belongs_to :udt, optional: true + belongs_to :open_transaction, class_name: "CkbTransaction" + belongs_to :closed_transaction, class_name: "CkbTransaction", optional: true - def open_transaction_hash - channel_outpoint[0..65] + validates :open_transaction_id, presence: true + + scope :open_channels, -> { where(closed_transaction_id: nil) } + + def outpoint_info + open_transaction.as_json(only: %i[tx_hash block_number block_timestamp transaction_fee]).merge( + { + funding_capacity: funding_cell.capacity, + funding_udt_amount: funding_cell.udt_amount, + funding_address: funding_cell.address_hash, + }, + ) end def udt_info udt&.as_json(only: %i[full_name symbol decimal icon_file]) end + + def funding_cell + open_transaction.outputs.includes(:lock_script).find_by( + lock_scripts: { code_hash: Settings.fiber_funding_code_hash }, + ) + end end # == Schema Information @@ -34,6 +52,8 @@ def udt_info # created_at :datetime not null # updated_at :datetime not null # udt_id :bigint +# open_transaction_id :bigint +# closed_transaction_id :bigint # # Indexes # diff --git a/app/models/fiber_graph_node.rb b/app/models/fiber_graph_node.rb index 4380c3639..543375320 100644 --- a/app/models/fiber_graph_node.rb +++ b/app/models/fiber_graph_node.rb @@ -22,6 +22,10 @@ def connected_node_ids node_ids = channel_links.pluck(:node1, :node2).flatten node_ids.uniq - [node_id] end + + def open_channels_count + channel_links.where(closed_transaction_id: nil).count + end end # == Schema Information diff --git a/app/views/api/v2/fiber/graph_channels/index.jbuilder b/app/views/api/v2/fiber/graph_channels/index.jbuilder index f38f4e022..f130eb241 100644 --- a/app/views/api/v2/fiber/graph_channels/index.jbuilder +++ b/app/views/api/v2/fiber/graph_channels/index.jbuilder @@ -1,6 +1,6 @@ json.data do json.fiber_graph_channels @channels do |channel| - json.(channel, :channel_outpoint, :node1, :node2, :chain_hash, :open_transaction_hash) + json.(channel, :channel_outpoint, :node1, :node2, :chain_hash, :outpoint_info) json.funding_tx_block_number channel.funding_tx_block_number.to_s json.funding_tx_index channel.funding_tx_index.to_s json.last_updated_timestamp channel.last_updated_timestamp.to_s diff --git a/app/views/api/v2/fiber/graph_nodes/index.jbuilder b/app/views/api/v2/fiber/graph_nodes/index.jbuilder index 1a4936d7e..03546373c 100644 --- a/app/views/api/v2/fiber/graph_nodes/index.jbuilder +++ b/app/views/api/v2/fiber/graph_nodes/index.jbuilder @@ -1,10 +1,11 @@ json.data do json.fiber_graph_nodes @nodes do |node| - json.(node, :alias, :node_id, :addresses, :timestamp, :chain_hash, :connected_node_ids) + json.(node, :alias, :node_id, :addresses, :timestamp, :chain_hash, :connected_node_ids, :open_channels_count) json.timestamp node.timestamp.to_s json.auto_accept_min_ckb_funding_amount node.auto_accept_min_ckb_funding_amount.to_s json.total_capacity node.total_capacity.to_s json.udt_cfg_infos node.udt_cfg_infos + json.channel_links_count node.channel_links.count end end json.meta do diff --git a/app/views/api/v2/fiber/graph_nodes/show.jbuilder b/app/views/api/v2/fiber/graph_nodes/show.jbuilder index 5b7db4ced..0796249dc 100644 --- a/app/views/api/v2/fiber/graph_nodes/show.jbuilder +++ b/app/views/api/v2/fiber/graph_nodes/show.jbuilder @@ -6,7 +6,7 @@ json.data do json.udt_cfg_infos @node.udt_cfg_infos json.fiber_graph_channels @graph_channels do |channel| - json.(channel, :channel_outpoint, :node1, :node2, :chain_hash, :open_transaction_hash) + json.(channel, :channel_outpoint, :node1, :node2, :chain_hash, :outpoint_info) json.funding_tx_block_number channel.funding_tx_block_number.to_s json.funding_tx_index channel.funding_tx_index.to_s json.last_updated_timestamp channel.last_updated_timestamp.to_s diff --git a/app/workers/fiber_graph_detect_worker.rb b/app/workers/fiber_graph_detect_worker.rb index 64c86282a..c60b491f3 100644 --- a/app/workers/fiber_graph_detect_worker.rb +++ b/app/workers/fiber_graph_detect_worker.rb @@ -3,12 +3,23 @@ class FiberGraphDetectWorker sidekiq_options queue: "fiber" def perform + # sync graph nodes and channels ["nodes", "channels"].each { fetch_graph_infos(_1) } + + # check channel is closed + FiberGraphChannel.open_channels.each do |channel| + funding_cell = channel.funding_cell + if funding_cell.consumed_by + channel.update(closed_transaction_id: funding_cell.consumed_by_id) + end + end end private def fetch_graph_infos(data_type) + return if ENV["FIBER_NODE_URL"].blank? + cursor = nil loop do @@ -22,79 +33,75 @@ def fetch_graph_infos(data_type) end def fetch_nodes(last_cursor) - return if ENV["FIBER_NODE_URL"].blank? - - data = rpc.graph_nodes(ENV["FIBER_NODE_URL"], { limit: "0x100", after: last_cursor }) - ApplicationRecord.transaction do - data["result"]["nodes"].each do |node| - node_attributes = { - alias: node["alias"], - node_id: node["node_id"], - addresses: node["addresses"], - timestamp: node["timestamp"].to_i(16), - chain_hash: node["chain_hash"], - auto_accept_min_ckb_funding_amount: node["auto_accept_min_ckb_funding_amount"], - } - - fiber_graph_node = FiberGraphNode.upsert(node_attributes, unique_by: %i[node_id], returning: %i[id]) - - cfg_info_attributes = [] - node["udt_cfg_infos"].each do |info| - udt = Udt.find_by(info["script"].symbolize_keys) - - if udt - cfg_info_attributes << { - fiber_graph_node_id: fiber_graph_node[0]["id"], - udt_id: udt.id, - auto_accept_amount: info["auto_accept_amount"].to_i(16), - } - end - end - - if cfg_info_attributes.present? - puts - FiberUdtCfgInfo.upsert_all(cfg_info_attributes, unique_by: %i[fiber_graph_node_id udt_id]) - end - end - end - - data["result"]["last_cursor"] + data = rpc.graph_nodes(ENV["FIBER_NODE_URL"], { limit: "0x64", after: last_cursor }) + ApplicationRecord.transaction { data.dig("result", "nodes").each { upsert_node_with_cfg_info(_1) } } + data.dig("result", "last_cursor") rescue StandardError => e Rails.logger.error("Error fetching nodes: #{e.message}") nil end def fetch_channels(last_cursor) - return if ENV["FIBER_NODE_URL"].blank? + data = rpc.graph_channels(ENV["FIBER_NODE_URL"], { limit: "0x64", after: last_cursor }) + channel_attributes = data.dig("result", "channels").map { build_channel_attributes(_1) }.compact + FiberGraphChannel.upsert_all(channel_attributes, unique_by: %i[channel_outpoint]) if channel_attributes.any? + data.dig("result", "last_cursor") + rescue StandardError => e + Rails.logger.error("Error fetching channels: #{e.message}") + nil + end - data = rpc.graph_channels(ENV["FIBER_NODE_URL"], { limit: "0x100", after: last_cursor }) - channel_attributes = data["result"]["channels"].map do |channel| - if (udt_type_script = channel["udt_type_script"]).present? - udt = Udt.find_by(type_hash: udt_type_script) - end + def upsert_node_with_cfg_info(node) + node_attributes = { + alias: node["alias"], + node_id: node["node_id"], + addresses: node["addresses"], + timestamp: node["timestamp"].to_i(16), + chain_hash: node["chain_hash"], + auto_accept_min_ckb_funding_amount: node["auto_accept_min_ckb_funding_amount"], + } + + fiber_graph_node = FiberGraphNode.upsert(node_attributes, unique_by: %i[node_id], returning: %i[id]) + + return unless fiber_graph_node && node["udt_cfg_infos"].present? + + cfg_info_attributes = node["udt_cfg_infos"].map do |info| + udt = Udt.find_by(info["script"].symbolize_keys) + next unless udt { - channel_outpoint: channel["channel_outpoint"], - funding_tx_block_number: channel["funding_tx_block_number"].to_i(16), - funding_tx_index: channel["funding_tx_index"].to_i(16), - node1: channel["node1"], - node2: channel["node2"], - last_updated_timestamp: channel["last_updated_timestamp"].to_i(16), - created_timestamp: channel["created_timestamp"], - node1_to_node2_fee_rate: channel["node1_to_node2_fee_rate"].to_i(16), - node2_to_node1_fee_rate: channel["node2_to_node1_fee_rate"].to_i(16), - capacity: channel["capacity"].to_i(16), - chain_hash: channel["chain_hash"], - udt_id: udt&.id, + fiber_graph_node_id: fiber_graph_node[0]["id"], + udt_id: udt.id, + auto_accept_amount: info["auto_accept_amount"].to_i(16), } - end + end.compact - FiberGraphChannel.upsert_all(channel_attributes, unique_by: %i[channel_outpoint]) if channel_attributes.any? + FiberUdtCfgInfo.upsert_all(cfg_info_attributes, unique_by: %i[fiber_graph_node_id udt_id]) if cfg_info_attributes.any? + end - data["result"]["last_cursor"] - rescue StandardError => e - Rails.logger.error("Error fetching channels: #{e.message}") - nil + def build_channel_attributes(channel) + if (udt_type_script = channel["udt_type_script"]).present? + udt = Udt.find_by(udt_type_script.symbolize_keys) + end + + channel_outpoint = channel["channel_outpoint"] + open_transaction = CkbTransaction.find_by(tx_hash: channel_outpoint[0..65]) + + { + channel_outpoint:, + funding_tx_block_number: channel["funding_tx_block_number"].to_i(16), + funding_tx_index: channel["funding_tx_index"].to_i(16), + node1: channel["node1"], + node2: channel["node2"], + last_updated_timestamp: channel["last_updated_timestamp"].to_i(16), + created_timestamp: channel["created_timestamp"], + node1_to_node2_fee_rate: channel["node1_to_node2_fee_rate"].to_i(16), + node2_to_node1_fee_rate: channel["node2_to_node1_fee_rate"].to_i(16), + capacity: channel["capacity"].to_i(16), + chain_hash: channel["chain_hash"], + open_transaction_id: open_transaction&.id, + udt_id: udt&.id, + } end def rpc diff --git a/config/settings.mainnet.yml b/config/settings.mainnet.yml index 1e47acde2..7e5072f9e 100644 --- a/config/settings.mainnet.yml +++ b/config/settings.mainnet.yml @@ -88,3 +88,6 @@ btc_time_code_hash: - "0x70d64497a075bd651e98ac030455ea200637ee325a12ad08aff03f1a117e5a62" single_use_lock_code_hash: "0x8290467a512e5b9a6b816469b0edabba1f4ac474e28ffdd604c2a7c76446bbaf" + +# fiber funding lock code hash +fiber_funding_code_hash: "" diff --git a/config/settings.testnet.yml b/config/settings.testnet.yml index 3fc0aac99..72c5c5f96 100644 --- a/config/settings.testnet.yml +++ b/config/settings.testnet.yml @@ -95,3 +95,6 @@ btc_time_code_hash: - "0x80a09eca26d77cea1f5a69471c59481be7404febf40ee90f886c36a948385b55" single_use_lock_code_hash: "0x8290467a512e5b9a6b816469b0edabba1f4ac474e28ffdd604c2a7c76446bbaf" + +# fiber funding lock code hash +fiber_funding_code_hash: "0x6c67887fe201ee0c7853f1682c0b77c0e6214044c156c7558269390a8afa6d7c" diff --git a/db/migrate/20241030023309_add_transaction_id_to_fiber_graph_channels.rb b/db/migrate/20241030023309_add_transaction_id_to_fiber_graph_channels.rb new file mode 100644 index 000000000..212379ae3 --- /dev/null +++ b/db/migrate/20241030023309_add_transaction_id_to_fiber_graph_channels.rb @@ -0,0 +1,8 @@ +class AddTransactionIdToFiberGraphChannels < ActiveRecord::Migration[7.0] + def change + change_table :fiber_graph_channels, bulk: true do |t| + t.bigint :open_transaction_id + t.bigint :closed_transaction_id + end + end +end diff --git a/db/structure.sql b/db/structure.sql index 3d4aa23c1..2467e44ba 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -1763,7 +1763,9 @@ CREATE TABLE public.fiber_graph_channels ( chain_hash character varying, created_at timestamp(6) without time zone NOT NULL, updated_at timestamp(6) without time zone NOT NULL, - udt_id bigint + udt_id bigint, + open_transaction_id bigint, + closed_transaction_id bigint ); @@ -6355,6 +6357,7 @@ INSERT INTO "schema_migrations" (version) VALUES ('20241009081935'), ('20241012014906'), ('20241023055256'), -('20241023063536'); +('20241023063536'), +('20241030023309'); diff --git a/lib/scheduler.rb b/lib/scheduler.rb index d31b5dcdf..ba21b8ba8 100644 --- a/lib/scheduler.rb +++ b/lib/scheduler.rb @@ -129,7 +129,6 @@ def call_worker(clz) end s.every "5m", overlap: false do - # call_worker FiberDetectWorker call_worker FiberGraphDetectWorker end From 7dc7e9b26b7832d2e4f7c65f74accdd0691c7e7f Mon Sep 17 00:00:00 2001 From: Rabbit Date: Mon, 4 Nov 2024 18:07:33 +0800 Subject: [PATCH 13/15] chore: add filter to fiber graph channels --- app/controllers/api/v2/fiber/graph_channels_controller.rb | 6 +++++- app/models/fiber_graph_node.rb | 5 +++-- app/views/api/v2/fiber/graph_nodes/index.jbuilder | 1 - 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/v2/fiber/graph_channels_controller.rb b/app/controllers/api/v2/fiber/graph_channels_controller.rb index 4e6108b0b..59fee4c0b 100644 --- a/app/controllers/api/v2/fiber/graph_channels_controller.rb +++ b/app/controllers/api/v2/fiber/graph_channels_controller.rb @@ -5,7 +5,11 @@ class GraphChannelsController < BaseController def index @page = params.fetch(:page, 1) @page_size = params.fetch(:page_size, FiberPeer.default_per_page) - @channels = FiberGraphChannel.all.page(@page).per(@page_size) + @channels = FiberGraphChannel.all + if params[:status] == "closed" + @channels = @channels.where.not(closed_transaction_id: nil) + end + @channels = @channels.page(@page).per(@page_size) end end end diff --git a/app/models/fiber_graph_node.rb b/app/models/fiber_graph_node.rb index 543375320..9dabb1e93 100644 --- a/app/models/fiber_graph_node.rb +++ b/app/models/fiber_graph_node.rb @@ -7,7 +7,8 @@ class FiberGraphNode < ApplicationRecord has_many :fiber_udt_cfg_infos, dependent: :delete_all def channel_links - FiberGraphChannel.where(node1: node_id).or(FiberGraphChannel.where(node2: node_id)) + FiberGraphChannel.where(node1: node_id).or(FiberGraphChannel.where(node2: node_id)). + where(closed_transaction_id: nil) end def udt_cfg_infos @@ -24,7 +25,7 @@ def connected_node_ids end def open_channels_count - channel_links.where(closed_transaction_id: nil).count + channel_links.count end end diff --git a/app/views/api/v2/fiber/graph_nodes/index.jbuilder b/app/views/api/v2/fiber/graph_nodes/index.jbuilder index 03546373c..55844fe0b 100644 --- a/app/views/api/v2/fiber/graph_nodes/index.jbuilder +++ b/app/views/api/v2/fiber/graph_nodes/index.jbuilder @@ -5,7 +5,6 @@ json.data do json.auto_accept_min_ckb_funding_amount node.auto_accept_min_ckb_funding_amount.to_s json.total_capacity node.total_capacity.to_s json.udt_cfg_infos node.udt_cfg_infos - json.channel_links_count node.channel_links.count end end json.meta do From 1898918258dedab63779d4d491a9f99e5d179d84 Mon Sep 17 00:00:00 2001 From: Miles Zhang Date: Mon, 18 Nov 2024 19:13:44 +0900 Subject: [PATCH 14/15] fix: daily_stattistic generator condition (#2288) Signed-off-by: Miles Zhang --- app/workers/charts/daily_statistic.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/charts/daily_statistic.rb b/app/workers/charts/daily_statistic.rb index d6010e453..3689e04d1 100644 --- a/app/workers/charts/daily_statistic.rb +++ b/app/workers/charts/daily_statistic.rb @@ -9,7 +9,7 @@ def perform(datetime = nil) datetime ||= 1.day.ago.beginning_of_day last_record = ::DailyStatistic.order(created_at_unixtimestamp: :desc).first start_time = Time.zone.at(last_record ? last_record.created_at_unixtimestamp : Block.find_by(number: 0).timestamp / 1000) - while start_time < datetime + while start_time <= datetime start_time += 1.day ActiveRecord::Base.connection.execute("SET statement_timeout = 0") Charts::DailyStatisticGenerator.new(start_time).call From 757cb72e9c102ee3758ba8d59b5b8efcbe7d18a9 Mon Sep 17 00:00:00 2001 From: Miles Zhang Date: Tue, 19 Nov 2024 10:52:41 +0900 Subject: [PATCH 15/15] Revert "fix: daily_stattistic generator condition" (#2290) Revert "fix: daily_stattistic generator condition (#2288)" This reverts commit 1898918258dedab63779d4d491a9f99e5d179d84. --- app/workers/charts/daily_statistic.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/charts/daily_statistic.rb b/app/workers/charts/daily_statistic.rb index 3689e04d1..d6010e453 100644 --- a/app/workers/charts/daily_statistic.rb +++ b/app/workers/charts/daily_statistic.rb @@ -9,7 +9,7 @@ def perform(datetime = nil) datetime ||= 1.day.ago.beginning_of_day last_record = ::DailyStatistic.order(created_at_unixtimestamp: :desc).first start_time = Time.zone.at(last_record ? last_record.created_at_unixtimestamp : Block.find_by(number: 0).timestamp / 1000) - while start_time <= datetime + while start_time < datetime start_time += 1.day ActiveRecord::Base.connection.execute("SET statement_timeout = 0") Charts::DailyStatisticGenerator.new(start_time).call