From 72835d10ded3965f0c8cf49e4eecba1452d20d59 Mon Sep 17 00:00:00 2001 From: Miles Zhang Date: Fri, 13 Dec 2024 14:09:30 +0900 Subject: [PATCH 1/2] fix: contract attrs may exist same deployed_cell_output_id but different role (#2338) Signed-off-by: Miles Zhang --- ...ze_contract_from_cell_dependency_worker.rb | 217 +++++++++++------- .../analyze_contract_from_start_block.rake | 138 ++++++----- 2 files changed, 215 insertions(+), 140 deletions(-) diff --git a/app/workers/analyze_contract_from_cell_dependency_worker.rb b/app/workers/analyze_contract_from_cell_dependency_worker.rb index a49a95d95..c8490c5c0 100644 --- a/app/workers/analyze_contract_from_cell_dependency_worker.rb +++ b/app/workers/analyze_contract_from_cell_dependency_worker.rb @@ -8,96 +8,159 @@ def perform cell_deps_out_points_attrs = Set.new contract_attrs = Set.new cell_deps_attrs = Set.new + contract_roles = Hash.new { |hash, key| hash[key] = {} } + + # 加载未分析的依赖数据 + dependencies = load_unanalyzed_dependencies + + # 按事务分组并逐组处理 + dependencies.each do |ckb_transaction_id, cell_deps| + ckb_transaction = CkbTransaction.find(ckb_transaction_id) + lock_scripts, type_scripts = analyze_scripts(ckb_transaction) + + process_cell_dependencies( + cell_deps, + lock_scripts, + type_scripts, + cell_deps_out_points_attrs, + contract_attrs, + cell_deps_attrs, + contract_roles, + ) + end - CellDependency.where(contract_analyzed: false).where.not(block_number: nil).order("block_number desc").limit(1000).each do |cell_dep| - cell_deps_attrs << { contract_analyzed: true, ckb_transaction_id: cell_dep.ckb_transaction_id, contract_cell_id: cell_dep.contract_cell_id, dep_type: cell_dep.dep_type } + # 持久化数据 + save_cell_deps_out_points(cell_deps_out_points_attrs) + save_contracts(contract_attrs, contract_roles) + save_cell_dependencies(cell_deps_attrs) + end - next if CellDepsOutPoint.where(contract_cell_id: cell_dep.contract_cell_id).exists? + private - ckb_transaction = CkbTransaction.find(cell_dep.ckb_transaction_id) + # 加载未分析的依赖数据 + def load_unanalyzed_dependencies + CellDependency.where(contract_analyzed: false). + where.not(block_number: nil). + order("block_number desc"). + limit(200). + group_by(&:ckb_transaction_id) + end - type_script_hashes = Set.new - lock_script_hashes = Set.new + # 分析脚本 + def analyze_scripts(ckb_transaction) + lock_scripts = {} + type_scripts = {} - cell_outputs = ckb_transaction.cell_outputs.includes(:type_script, :lock_script).to_a - cell_inputs = ckb_transaction.cell_inputs.includes(:previous_cell_output).map(&:previous_cell_output) - cell_inputs.each do |input| - lock_script_hashes << input.lock_script.code_hash - type_script_hashes << input.type_script.code_hash if input.type_script - end + cell_outputs = ckb_transaction.cell_outputs.includes(:type_script).to_a + cell_inputs = ckb_transaction.cell_inputs.includes(:previous_cell_output).map(&:previous_cell_output) - cell_outputs.each do |output| - lock_script_hashes << output.lock_script.code_hash - type_script_hashes << output.type_script.code_hash if output.type_script + (cell_inputs + cell_outputs).each do |cell| + lock_scripts[cell.lock_script.code_hash] = cell.lock_script.hash_type + if cell.type_script + type_scripts[cell.type_script.code_hash] = cell.type_script.hash_type end + end + + [lock_scripts, type_scripts] + end + + # 处理每个依赖 + def process_cell_dependencies(cell_deps, lock_scripts, type_scripts, out_points_attrs, contract_attrs, cell_deps_attrs, contract_roles) + cell_deps.each do |cell_dep| + cell_deps_attrs << { + contract_analyzed: true, + ckb_transaction_id: cell_dep.ckb_transaction_id, + contract_cell_id: cell_dep.contract_cell_id, + dep_type: cell_dep.dep_type, + } + next if Contract.joins(:cell_deps_out_points).where(cell_deps_out_points: { contract_cell_id: cell_dep.contract_cell_id }).exists? case cell_dep.dep_type when "code" - cell_output = cell_dep.cell_output - cell_deps_out_points_attrs << { - tx_hash: cell_output.tx_hash, - cell_index: cell_output.cell_index, - deployed_cell_output_id: cell_output.id, - contract_cell_id: cell_output.id, - } - - is_lock_script = cell_output.type_script&.script_hash.in?(lock_script_hashes) || cell_output.data_hash.in?(lock_script_hashes) - is_type_script = cell_output.type_script&.script_hash.in?(type_script_hashes) || cell_output.data_hash.in?(type_script_hashes) - - if is_lock_script || is_type_script - contract_attrs << - { - type_hash: cell_output.type_script&.script_hash, - data_hash: cell_output.data_hash, - deployed_cell_output_id: cell_output.id, - is_type_script:, - is_lock_script:, - deployed_args: cell_output.type_script&.args, - } - end - + process_code_dep(cell_dep, lock_scripts, type_scripts, out_points_attrs, contract_attrs, contract_roles) when "dep_group" - # when the type of cell_dep is "dep_group", it means the cell specified by the `out_point` is a list of out points to the actual referred contract cells - mid_cell = cell_dep.cell_output - - binary_data = mid_cell.binary_data - # parse the actual list of out points from the data field of the cell - out_points_count = binary_data[0, 4].unpack("L<") - # iterate over the out point list and append actual referred contract cells to cell dependencies_attrs - 0.upto(out_points_count[0] - 1) do |i| - part_tx_hash, cell_index = binary_data[4 + i * 36, 36].unpack("H64L<") - - tx_hash = "0x#{part_tx_hash}" - cell_output = CellOutput.find_by_pointer tx_hash, cell_index - cell_deps_out_points_attrs << { - tx_hash:, - cell_index:, - deployed_cell_output_id: cell_output.id, - contract_cell_id: mid_cell.id, - } - - is_lock_script = cell_output.type_script&.script_hash.in?(lock_script_hashes) || cell_output.data_hash.in?(lock_script_hashes) - is_type_script = cell_output.type_script&.script_hash.in?(type_script_hashes) || cell_output.data_hash.in?(type_script_hashes) - - if is_lock_script || is_type_script - contract_attrs << - { - type_hash: cell_output.type_script&.script_hash, - data_hash: cell_output.data_hash, - deployed_cell_output_id: cell_output.id, - deployed_args: cell_output.type_script&.args, - is_type_script:, - is_lock_script:, - } - end - end + process_dep_group(cell_dep, lock_scripts, type_scripts, out_points_attrs, contract_attrs, contract_roles) end end - if cell_deps_out_points_attrs.any? - CellDepsOutPoint.upsert_all(cell_deps_out_points_attrs, - unique_by: %i[contract_cell_id deployed_cell_output_id]) + end + + # 处理 "code" 类型依赖 + def process_code_dep(cell_dep, lock_scripts, type_scripts, out_points_attrs, contract_attrs, contract_roles) + cell_output = cell_dep.cell_output + out_points_attrs << { + tx_hash: cell_output.tx_hash, + cell_index: cell_output.cell_index, + deployed_cell_output_id: cell_output.id, + contract_cell_id: cell_output.id, + } + + update_contract_roles(cell_output, lock_scripts, type_scripts, contract_roles) + + if contract_roles[cell_output.id][:is_lock_script] || contract_roles[cell_output.id][:is_type_script] + contract_attrs << build_contract_attr(cell_output, lock_scripts, type_scripts) end - Contract.upsert_all(contract_attrs, unique_by: %i[deployed_cell_output_id]) if contract_attrs.any? - CellDependency.upsert_all(cell_deps_attrs, unique_by: %i[ckb_transaction_id contract_cell_id dep_type], update_only: :contract_analyzed) + end + + # 处理 "dep_group" 类型依赖 + def process_dep_group(cell_dep, lock_scripts, type_scripts, out_points_attrs, contract_attrs, contract_roles) + mid_cell = cell_dep.cell_output + binary_data = mid_cell.binary_data + out_points_count = binary_data[0, 4].unpack1("L<") + + 0.upto(out_points_count - 1) do |i| + part_tx_hash, cell_index = binary_data[4 + i * 36, 36].unpack("H64L<") + tx_hash = "0x#{part_tx_hash}" + cell_output = CellOutput.find_by_pointer(tx_hash, cell_index) + + out_points_attrs << { + tx_hash:, + cell_index:, + deployed_cell_output_id: cell_output.id, + contract_cell_id: mid_cell.id, + } + + update_contract_roles(cell_output, lock_scripts, type_scripts, contract_roles) + + if contract_roles[cell_output.id][:is_lock_script] || contract_roles[cell_output.id][:is_type_script] + contract_attrs << build_contract_attr(cell_output, lock_scripts, type_scripts) + end + end + end + + # 更新 contract_roles + def update_contract_roles(cell_output, lock_scripts, type_scripts, contract_roles) + is_lock_script = (lock_scripts[cell_output.data_hash] || lock_scripts[cell_output.type_script&.script_hash]).present? + is_type_script = (type_scripts[cell_output.data_hash] || type_scripts[cell_output.type_script&.script_hash]).present? + data_type = lock_scripts[cell_output.data_hash] || type_scripts[cell_output.data_hash] + + contract_roles[cell_output.id][:is_lock_script] ||= is_lock_script + contract_roles[cell_output.id][:is_type_script] ||= is_type_script + contract_roles[cell_output.id][:hash_type] ||= data_type + end + + # 构建单个合约属性 + def build_contract_attr(cell_output, _lock_scripts, _type_scripts) + { + type_hash: cell_output.type_script&.script_hash, + data_hash: cell_output.data_hash, + deployed_cell_output_id: cell_output.id, + deployed_args: cell_output.type_script&.args, + } + end + + # 保存数据 + def save_cell_deps_out_points(attrs) + CellDepsOutPoint.upsert_all(attrs.to_a, unique_by: %i[contract_cell_id deployed_cell_output_id]) if attrs.any? + end + + def save_contracts(attrs, roles) + return if attrs.empty? + + new_attrs = attrs.map { |attr| attr.merge(roles[attr[:deployed_cell_output_id]]) } + Contract.upsert_all(new_attrs, unique_by: %i[deployed_cell_output_id]) + end + + def save_cell_dependencies(attrs) + CellDependency.upsert_all(attrs.to_a, unique_by: %i[ckb_transaction_id contract_cell_id dep_type], update_only: :contract_analyzed) if attrs.any? end end diff --git a/lib/tasks/migration/analyze_contract_from_start_block.rake b/lib/tasks/migration/analyze_contract_from_start_block.rake index 87fde67a8..53b8cf48f 100644 --- a/lib/tasks/migration/analyze_contract_from_start_block.rake +++ b/lib/tasks/migration/analyze_contract_from_start_block.rake @@ -5,76 +5,46 @@ namespace :migration do cell_deps_out_points_attrs = Set.new contract_attrs = Set.new cell_deps_attrs = Set.new + contract_roles = Hash.new { |hash, key| hash[key] = {} } - CellDependency.where(contract_analyzed: false).where.not(block_number: nil).limit(1000).each do |cell_dep| - cell_deps_attrs << { contract_analyzed: true, ckb_transaction_id: cell_dep.ckb_transaction_id, contract_cell_id: cell_dep.contract_cell_id, dep_type: cell_dep.dep_type } + CellDependency.where(contract_analyzed: false).where.not(block_number: nil).limit(1000).group_by do |cell_dep| + cell_dep.ckb_transaction_id + end.each do |ckb_transaction_id, cell_deps| + ckb_transaction = CkbTransaction.find(ckb_transaction_id) + type_scripts = Hash.new + lock_scripts = Hash.new - next if CellDepsOutPoint.where(contract_cell_id: cell_dep.contract_cell_id).exists? - - ckb_transaction = CkbTransaction.find(cell_dep.ckb_transaction_id) - - type_script_hashes = Set.new - lock_script_hashes = Set.new - - cell_outputs = ckb_transaction.cell_outputs.includes(:type_script, :lock_script).to_a + cell_outputs = ckb_transaction.cell_outputs.includes(:type_script).to_a cell_inputs = ckb_transaction.cell_inputs.includes(:previous_cell_output).map(&:previous_cell_output) cell_inputs.each do |input| - lock_script_hashes << input.lock_script.code_hash - type_script_hashes << input.type_script.code_hash if input.type_script + lock_scripts[input.lock_script.code_hash] = input.lock_script.hash_type + type_scripts[input.type_script.code_hash] = input.type_script.hash_type if input.type_script end - cell_outputs.each do |output| - lock_script_hashes << output.lock_script.code_hash - type_script_hashes << output.type_script.code_hash if output.type_script + lock_scripts[output.lock_script.code_hash] = output.lock_script.hash_type + type_scripts[output.type_script.code_hash] = output.type_script.hash_type if output.type_script end - case cell_dep.dep_type - when "code" - cell_output = cell_dep.cell_output - cell_deps_out_points_attrs << { - tx_hash: cell_output.tx_hash, - cell_index: cell_output.cell_index, - deployed_cell_output_id: cell_output.id, - contract_cell_id: cell_output.id, - } - - is_lock_script = cell_output.type_script&.script_hash.in?(lock_script_hashes) || cell_output.data_hash.in?(lock_script_hashes) - is_type_script = cell_output.type_script&.script_hash.in?(type_script_hashes) || cell_output.data_hash.in?(type_script_hashes) - - if is_lock_script || is_type_script - contract_attrs << - { - type_hash: cell_output.type_script&.script_hash, - data_hash: cell_output.data_hash, - deployed_cell_output_id: cell_output.id, - is_type_script:, - is_lock_script:, - deployed_args: cell_output.type_script&.args, - } - end - - when "dep_group" - # when the type of cell_dep is "dep_group", it means the cell specified by the `out_point` is a list of out points to the actual referred contract cells - mid_cell = cell_dep.cell_output + cell_deps.each do |cell_dep| + cell_deps_attrs << { contract_analyzed: true, ckb_transaction_id: cell_dep.ckb_transaction_id, contract_cell_id: cell_dep.contract_cell_id, dep_type: cell_dep.dep_type } + next if Contract.joins(:cell_deps_out_points).where(cell_deps_out_points: { contract_cell_id: cell_dep.contract_cell_id }).exists? - binary_data = mid_cell.binary_data - # parse the actual list of out points from the data field of the cell - out_points_count = binary_data[0, 4].unpack("L<") - # iterate over the out point list and append actual referred contract cells to cell dependencies_attrs - 0.upto(out_points_count[0] - 1) do |i| - part_tx_hash, cell_index = binary_data[4 + i * 36, 36].unpack("H64L<") - - tx_hash = "0x#{part_tx_hash}" - cell_output = CellOutput.find_by_pointer tx_hash, cell_index + case cell_dep.dep_type + when "code" + cell_output = cell_dep.cell_output cell_deps_out_points_attrs << { - tx_hash:, - cell_index:, + tx_hash: cell_output.tx_hash, + cell_index: cell_output.cell_index, deployed_cell_output_id: cell_output.id, - contract_cell_id: mid_cell.id, + contract_cell_id: cell_output.id, } - is_lock_script = cell_output.type_script&.script_hash.in?(lock_script_hashes) || cell_output.data_hash.in?(lock_script_hashes) - is_type_script = cell_output.type_script&.script_hash.in?(type_script_hashes) || cell_output.data_hash.in?(type_script_hashes) + is_lock_script = (lock_scripts[cell_output.data_hash] || lock_scripts[cell_output.type_script&.script_hash]).present? + is_type_script = (type_scripts[cell_output.data_hash] || type_scripts[cell_output.type_script&.script_hash]).present? + data_type = lock_scripts[cell_output.data_hash] || type_scripts[cell_output.data_hash] + contract_roles[cell_output.id][:is_lock_script] ||= is_lock_script + contract_roles[cell_output.id][:is_type_script] ||= is_type_script + contract_roles[cell_output.id][:hash_type] ||= data_type if is_lock_script || is_type_script contract_attrs << @@ -83,23 +53,65 @@ namespace :migration do data_hash: cell_output.data_hash, deployed_cell_output_id: cell_output.id, deployed_args: cell_output.type_script&.args, - is_type_script:, - is_lock_script:, } end + + when "dep_group" + # when the type of cell_dep is "dep_group", it means the cell specified by the `out_point` is a list of out points to the actual referred contract cells + mid_cell = cell_dep.cell_output + + binary_data = mid_cell.binary_data + # parse the actual list of out points from the data field of the cell + out_points_count = binary_data[0, 4].unpack("L<") + # iterate over the out point list and append actual referred contract cells to cell dependencies_attrs + 0.upto(out_points_count[0] - 1) do |i| + part_tx_hash, cell_index = binary_data[4 + i * 36, 36].unpack("H64L<") + + tx_hash = "0x#{part_tx_hash}" + cell_output = CellOutput.find_by_pointer tx_hash, cell_index + cell_deps_out_points_attrs << { + tx_hash:, + cell_index:, + deployed_cell_output_id: cell_output.id, + contract_cell_id: mid_cell.id, + } + + is_lock_script = (lock_scripts[cell_output.data_hash] || lock_scripts[cell_output.type_script&.script_hash]).present? + is_type_script = (type_scripts[cell_output.data_hash] || type_scripts[cell_output.type_script&.script_hash]).present? + hash_type = lock_scripts[cell_output.data_hash] || type_scripts[cell_output.data_hash] + contract_roles[cell_output.id][:is_lock_script] ||= is_lock_script + contract_roles[cell_output.id][:is_type_script] ||= is_type_script + contract_roles[cell_output.id][:hash_type] ||= hash_type + + if is_lock_script || is_type_script + contract_attrs << + { + type_hash: cell_output.type_script&.script_hash, + data_hash: cell_output.data_hash, + + deployed_cell_output_id: cell_output.id, + deployed_args: cell_output.type_script&.args, + } + end + end end end end - if cell_deps_out_points_attrs.any? CellDepsOutPoint.upsert_all(cell_deps_out_points_attrs, unique_by: %i[contract_cell_id deployed_cell_output_id]) end - Contract.upsert_all(contract_attrs, unique_by: %i[deployed_cell_output_id]) if contract_attrs.any? + # some contract in a cell may be lock script but in another cell may be type script + if contract_attrs.any? + new_contract_attrs = + contract_attrs.map do |attr| + attr.merge(contract_roles[attr[:deployed_cell_output_id]]) + end + puts new_contract_attrs + Contract.upsert_all(new_contract_attrs, unique_by: %i[deployed_cell_output_id]) + end CellDependency.upsert_all(cell_deps_attrs, unique_by: %i[ckb_transaction_id contract_cell_id dep_type], update_only: :contract_analyzed) - puts cell_deps_attrs.to_a.last[:ckb_transaction_id] - break unless CellDependency.where(contract_analyzed: false).where.not(block_number: nil).exists? end puts "DONE" From 06d7da01ba6134d57e42fa0f79781f1efb44d517 Mon Sep 17 00:00:00 2001 From: Miles Zhang Date: Sat, 14 Dec 2024 10:48:16 +0900 Subject: [PATCH 2/2] feat: remove cell_dependency desc order (#2342) Signed-off-by: Miles Zhang --- app/workers/analyze_contract_from_cell_dependency_worker.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/app/workers/analyze_contract_from_cell_dependency_worker.rb b/app/workers/analyze_contract_from_cell_dependency_worker.rb index c8490c5c0..d87544053 100644 --- a/app/workers/analyze_contract_from_cell_dependency_worker.rb +++ b/app/workers/analyze_contract_from_cell_dependency_worker.rb @@ -41,7 +41,6 @@ def perform def load_unanalyzed_dependencies CellDependency.where(contract_analyzed: false). where.not(block_number: nil). - order("block_number desc"). limit(200). group_by(&:ckb_transaction_id) end