From f9d423b66a2f0bbca30227c1663fbeb2a2787e0d Mon Sep 17 00:00:00 2001 From: NanZhang Date: Tue, 28 Nov 2023 15:36:05 +0800 Subject: [PATCH 1/3] feat: add address pending transactions (#1514) --- ...address_pending_transactions_controller.rb | 82 +++++ app/models/address.rb | 6 - config/routes.rb | 1 + .../fix_address_balance_occupied.rake | 2 - ...ss_pending_transactions_controller_test.rb | 346 ++++++++++++++++++ test/factories/address.rb | 23 +- 6 files changed, 446 insertions(+), 14 deletions(-) create mode 100644 app/controllers/api/v1/address_pending_transactions_controller.rb create mode 100644 test/controllers/api/v1/address_pending_transactions_controller_test.rb diff --git a/app/controllers/api/v1/address_pending_transactions_controller.rb b/app/controllers/api/v1/address_pending_transactions_controller.rb new file mode 100644 index 000000000..cbbda706d --- /dev/null +++ b/app/controllers/api/v1/address_pending_transactions_controller.rb @@ -0,0 +1,82 @@ +module Api + module V1 + class AddressPendingTransactionsController < ApplicationController + before_action :validate_query_params + before_action :validate_pagination_params, :pagination_params + before_action :find_address + + def show + expires_in 10.seconds, public: true, must_revalidate: true, stale_while_revalidate: 10.seconds + + ckb_transactions = @address.ckb_transactions.tx_pending + ckb_transactions_ids = CellInput.where(ckb_transaction_id: ckb_transactions.ids). + where.not(previous_cell_output_id: nil, from_cell_base: false). + distinct.pluck(:ckb_transaction_id) + @ckb_transactions = CkbTransaction.where(id: ckb_transactions_ids). + order(transactions_ordering).page(@page).per(@page_size) + + render json: serialized_ckb_transactions + end + + private + + def validate_query_params + validator = Validations::Address.new(params) + + if validator.invalid? + errors = validator.error_object[:errors] + status = validator.error_object[:status] + + render json: errors, status: status + end + end + + def pagination_params + @page = params[:page] || 1 + @page_size = params[:page_size] || CkbTransaction.default_per_page + end + + def find_address + @address = Address.find_address!(params[:id]) + raise Api::V1::Exceptions::AddressNotFoundError if @address.is_a?(NullAddress) + end + + def transactions_ordering + sort, order = params.fetch(:sort, "id.desc").split(".", 2) + sort = case sort + when "time" then "block_timestamp" + else "id" + end + + order = order.match?(/^(asc|desc)$/i) ? order : "asc" + + "#{sort} #{order} NULLS LAST" + end + + def serialized_ckb_transactions + options = FastJsonapi::PaginationMetaGenerator.new( + request: request, + records: @ckb_transactions, + page: @page, + page_size: @page_size + ).call + ckb_transaction_serializer = CkbTransactionsSerializer.new( + @ckb_transactions, + options.merge(params: { previews: true, address: @address }) + ) + + if QueryKeyUtils.valid_address?(params[:id]) + if @address.address_hash == @address.query_address + ckb_transaction_serializer.serialized_json + else + ckb_transaction_serializer.serialized_json.gsub( + @address.address_hash, @address.query_address + ) + end + else + ckb_transaction_serializer.serialized_json + end + end + end + end +end diff --git a/app/models/address.rb b/app/models/address.rb index f56370439..438e1c61f 100644 --- a/app/models/address.rb +++ b/app/models/address.rb @@ -129,12 +129,6 @@ def self.find_address!(query_key) end def self.cached_find(query_key) - cache_key = query_key - - unless QueryKeyUtils.valid_hex?(query_key) - cache_key = CkbUtils.parse_address(query_key).script.compute_hash - end - address = if QueryKeyUtils.valid_hex?(query_key) find_by(lock_hash: query_key) diff --git a/config/routes.rb b/config/routes.rb index c8dba6a47..46f05ca0d 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -64,6 +64,7 @@ resources :distribution_data, only: :show resources :monetary_data, only: :show resources :udt_verifications, only: :update + resources :address_pending_transactions, only: :show end end draw "v2" diff --git a/lib/tasks/migration/fix_address_balance_occupied.rake b/lib/tasks/migration/fix_address_balance_occupied.rake index b468d9fd7..7d2997151 100644 --- a/lib/tasks/migration/fix_address_balance_occupied.rake +++ b/lib/tasks/migration/fix_address_balance_occupied.rake @@ -4,8 +4,6 @@ namespace :migration do pr_merged_datetime = DateTime.new(2022,7,23,0,0,0) addresses = Address.where("updated_at > ?", pr_merged_datetime).order(id: :asc) addresses.find_each do |address| - puts "Address ID: #{address.id}" - occupied = address.cell_outputs.live.occupied.sum(:capacity) address.update(balance_occupied: occupied) end diff --git a/test/controllers/api/v1/address_pending_transactions_controller_test.rb b/test/controllers/api/v1/address_pending_transactions_controller_test.rb new file mode 100644 index 000000000..d9958fa57 --- /dev/null +++ b/test/controllers/api/v1/address_pending_transactions_controller_test.rb @@ -0,0 +1,346 @@ +require "test_helper" + +module Api + module V1 + class AddressPendingTransactionsControllerTest < ActionDispatch::IntegrationTest + test "should get success code when call show" do + address = create(:address, :with_pending_transactions) + + valid_get api_v1_address_pending_transaction_url(address.address_hash) + + assert_response :success + end + + test "should set right content type when call show" do + address = create(:address, :with_pending_transactions) + + valid_get api_v1_address_pending_transaction_url(address.address_hash) + + assert_equal "application/vnd.api+json", response.media_type + end + + test "should respond with 415 Unsupported Media Type when Content-Type is wrong" do + address = create(:address, :with_pending_transactions) + + get api_v1_address_pending_transaction_url(address.address_hash), + headers: { "Content-Type": "text/plain" } + + assert_equal 415, response.status + end + + test "should respond with error object when Content-Type is wrong" do + address = create(:address, :with_pending_transactions) + error_object = Api::V1::Exceptions::InvalidContentTypeError.new + response_json = RequestErrorSerializer.new([error_object], + message: error_object.title).serialized_json + + get api_v1_address_pending_transaction_url(address.address_hash), + headers: { "Content-Type": "text/plain" } + + assert_equal response_json, response.body + end + + test "should respond with 406 Not Acceptable when Accept is wrong" do + address = create(:address, :with_pending_transactions) + + get api_v1_address_pending_transaction_url(address.address_hash), + headers: { + "Content-Type": "application/vnd.api+json", + "Accept": "application/json" + } + + assert_equal 406, response.status + end + + test "should respond with error object when Accept is wrong" do + address = create(:address, :with_pending_transactions) + error_object = Api::V1::Exceptions::InvalidAcceptError.new + response_json = RequestErrorSerializer.new( + [error_object], + message: error_object.title + ).serialized_json + + get api_v1_address_pending_transaction_url(address.address_hash), + headers: { + "Content-Type": "application/vnd.api+json", + "Accept": "application/json" + } + + assert_equal response_json, response.body + end + + test "should return error object when id is not a address hash" do + error_object = Api::V1::Exceptions::AddressHashInvalidError.new + response_json = RequestErrorSerializer.new([error_object], + message: error_object.title).serialized_json + + valid_get api_v1_address_pending_transaction_url("9034fwefwef") + + assert_equal response_json, response.body + end + + test "should return corresponding ckb transactions with given address hash" do + page = 1 + page_size = 10 + address = create(:address, :with_pending_transactions) + ckb_transactions = address.ckb_transactions.order(id: :desc).page(page).per(page_size) + + valid_get api_v1_address_pending_transaction_url(address.address_hash) + + options = FastJsonapi::PaginationMetaGenerator.new( + request: request, + records: ckb_transactions, + page: page, + page_size: page_size + ).call + + assert_equal CkbTransactionsSerializer.new( + ckb_transactions, + options.merge( + params: { + previews: true, + address: address + } + ) + ).serialized_json, response.body + end + + test "should return corresponding ckb transactions with given lock hash" do + page = 1 + page_size = 10 + address = create(:address, :with_pending_transactions) + ckb_transactions = address.ckb_transactions.order(block_timestamp: :desc).page(page).per(page_size) + + valid_get api_v1_address_pending_transaction_url(address.lock_hash) + + options = FastJsonapi::PaginationMetaGenerator.new( + request: request, + records: ckb_transactions, + page: page, + page_size: page_size + ).call + + assert_equal CkbTransactionsSerializer.new( + ckb_transactions, + options.merge( + params: { + previews: true, + address: address + } + ) + ).serialized_json, response.body + end + + test "should contain right keys in the serialized object when call show" do + address = create(:address, :with_pending_transactions) + + valid_get api_v1_address_pending_transaction_url(address.address_hash) + + response_tx_transaction = json["data"].first + + assert_equal %w( + block_number + block_timestamp + display_inputs + display_inputs_count + display_outputs + display_outputs_count + income + is_cellbase + transaction_hash + ).sort, response_tx_transaction["attributes"].keys.sort + end + + test "should return error object when no records found by id" do + error_object = Api::V1::Exceptions::AddressNotFoundError.new + response_json = RequestErrorSerializer.new([error_object], + message: error_object.title).serialized_json + + valid_get api_v1_address_pending_transaction_url("ckt1qyqrdsefa43s6m882pcj53m4gdnj4k440axqswmu83") + + assert_equal response_json, response.body + end + + test "should return error object when page param is invalid" do + address = create(:address, :with_pending_transactions) + error_object = Api::V1::Exceptions::PageParamError.new + response_json = RequestErrorSerializer.new([error_object], + message: error_object.title).serialized_json + + valid_get api_v1_address_pending_transaction_url(address.address_hash), + params: { page: "aaa" } + + assert_equal response_json, response.body + end + + test "should return error object when page size param is invalid" do + address = create(:address, :with_pending_transactions) + error_object = Api::V1::Exceptions::PageSizeParamError.new + response_json = RequestErrorSerializer.new([error_object], + message: error_object.title).serialized_json + + valid_get api_v1_address_pending_transaction_url(address.address_hash), + params: { page_size: "aaa" } + + assert_equal response_json, response.body + end + + test "should return error object when page and page size param are invalid" do + errors = [] + address = create(:address, :with_pending_transactions) + errors << Api::V1::Exceptions::PageParamError.new + errors << Api::V1::Exceptions::PageSizeParamError.new + response_json = RequestErrorSerializer.new(errors, + message: errors.first.title).serialized_json + + valid_get api_v1_address_pending_transaction_url(address.address_hash), + params: { page: "bbb", page_size: "aaa" } + + assert_equal response_json, response.body + end + + test "should return 10 records when page and page_size are not set" do + address = create(:address, :with_pending_transactions, transactions_count: 15) + + valid_get api_v1_address_pending_transaction_url(address.address_hash) + + assert_equal 10, json["data"].size + end + + test "should return corresponding page's records when page is set and page_size is not set" do + page = 2 + page_size = 10 + address = create(:address, :with_pending_transactions, transactions_count: 30) + address_ckb_transactions = address.custom_ckb_transactions. + order("id desc NULLS LAST"). + page(page). + per(page_size) + valid_get api_v1_address_pending_transaction_url(address.address_hash), params: { page: page } + + options = FastJsonapi::PaginationMetaGenerator.new(request: request, + records: address_ckb_transactions, + page: page, + page_size: page_size).call + response_transaction = CkbTransactionsSerializer.new( + address_ckb_transactions, options.merge(params: { + previews: true, + address: address }) + ).serialized_json + + assert_equal response_transaction, response.body + assert_equal page_size, json["data"].size + end + + test "should return the corresponding transactions under the address when page is not set and page_size is set" do + page = 1 + page_size = 12 + address = create(:address, :with_pending_transactions, transactions_count: 15) + address_ckb_transactions = address.ckb_transactions.order("id desc NULLS LAST").page(page).per(page_size) + + valid_get api_v1_address_pending_transaction_url(address.address_hash), + params: { page_size: page_size } + + options = FastJsonapi::PaginationMetaGenerator.new(request: request, + records: address_ckb_transactions, + page: page, + page_size: page_size).call + response_transaction = CkbTransactionsSerializer.new( + address_ckb_transactions, options.merge(params: { + previews: true, + address: address + }) + ).serialized_json + + assert_equal response_transaction, response.body + assert_equal page_size, json["data"].size + end + + test "should return the corresponding transactions when page and page_size are set" do + page = 2 + page_size = 5 + address = create(:address, :with_pending_transactions, transactions_count: 30) + address_ckb_transactions = address.ckb_transactions.order("id desc NULLS LAST").page(page).per(page_size) + + valid_get api_v1_address_pending_transaction_url(address.address_hash), + params: { page: page, page_size: page_size } + + options = FastJsonapi::PaginationMetaGenerator.new(request: request, + records: address_ckb_transactions, + page: page, + page_size: page_size).call + response_transaction = CkbTransactionsSerializer.new( + address_ckb_transactions, options.merge(params: { + previews: true, + address: address + }) + ).serialized_json + + assert_equal response_transaction, response.body + end + + test "should return empty array when there is no record under the address" do + page = 2 + page_size = 5 + address = create(:address, :with_pending_transactions) + address_ckb_transactions = address.ckb_transactions.order("id desc NULLS LAST").page(page).per(page_size) + + valid_get api_v1_address_pending_transaction_url(address.address_hash), + params: { page: page, page_size: page_size } + + options = FastJsonapi::PaginationMetaGenerator.new(request: request, + records: address_ckb_transactions, + page: page, + page_size: page_size).call + response_transaction = CkbTransactionsSerializer.new( + address_ckb_transactions, options.merge(params: { + previews: true, + address: address + }) + ).serialized_json + + assert_equal [], json["data"] + assert_equal response_transaction, response.body + end + + test "should return meta that contained total in response body" do + address = create(:address, :with_pending_transactions, transactions_count: 3) + + valid_get api_v1_address_pending_transaction_url(address.address_hash) + + assert_equal 3, json.dig("meta", "total") + end + + test "should return up to ten display_inputs" do + address = create(:address) + block = create(:block, :with_block_hash) + create(:pending_transaction, + :with_multiple_inputs_and_outputs, block: block, contained_address_ids: [address.id]) + + valid_get api_v1_address_pending_transaction_url(address.address_hash) + + assert_equal 10, + json["data"].first.dig("attributes", + "display_inputs").count + assert_equal [true], json["data"].first.dig("attributes", "display_inputs").map { |input| + input.key?("from_cellbase") + }.uniq + end + + test "should return up to ten display_outputs" do + address = create(:address) + block = create(:block, :with_block_hash) + create(:pending_transaction, + :with_multiple_inputs_and_outputs, block: block, contained_address_ids: [address.id]) + + valid_get api_v1_address_pending_transaction_url(address.address_hash) + + assert_equal 10, + json["data"].first.dig("attributes", + "display_outputs").count + assert_equal [false], json["data"].first.dig("attributes", "display_outputs").map { |input| + input.key?("from_cellbase") + }.uniq + end + end + end +end diff --git a/test/factories/address.rb b/test/factories/address.rb index d21a7d053..6a0f2d8c0 100644 --- a/test/factories/address.rb +++ b/test/factories/address.rb @@ -1,7 +1,8 @@ FactoryBot.define do factory :address do address_hash do - script = CKB::Types::Script.new(code_hash: Settings.secp_cell_type_hash, args: "0x#{SecureRandom.hex(20)}", hash_type: "type") + script = CKB::Types::Script.new(code_hash: Settings.secp_cell_type_hash, args: "0x#{SecureRandom.hex(20)}", + hash_type: "type") CKB::Address.new(script).generate end @@ -40,11 +41,21 @@ ckb_transactions << create(:ckb_transaction, block: block, block_timestamp: Time.current.to_i + i) end - # ckb_transactions.each do |tx| - # tx.contained_address_ids << address.id - # tx.save - # end - # binding.pry + AccountBook.upsert_all ckb_transactions.map { |t| { address_id: address.id, ckb_transaction_id: t.id } } + address.update(ckb_transactions_count: address.ckb_transactions.count) + end + end + + trait :with_pending_transactions do + ckb_transactions_count { 3 } + after(:create) do |address, evaluator| + block = create(:block, :with_block_hash) + ckb_transactions = [] + evaluator.transactions_count.times do |i| + ckb_transactions << create(:pending_transaction, :with_multiple_inputs_and_outputs, block: block, + block_timestamp: Time.current.to_i + i) + end + AccountBook.upsert_all ckb_transactions.map { |t| { address_id: address.id, ckb_transaction_id: t.id } } address.update(ckb_transactions_count: address.ckb_transactions.count) end From 8ee1a354697481440e58a2094fe717df964a7ef0 Mon Sep 17 00:00:00 2001 From: Rabbit Date: Wed, 29 Nov 2023 11:30:38 +0800 Subject: [PATCH 2/3] fix: filter out invalid addresses in das accounts query (#1518) --- .../api/v2/das_accounts_controller.rb | 33 +++++++++++-------- .../api/v2/das_accounts_controller_test.rb | 18 ++++++---- 2 files changed, 30 insertions(+), 21 deletions(-) diff --git a/app/controllers/api/v2/das_accounts_controller.rb b/app/controllers/api/v2/das_accounts_controller.rb index dd78c01f4..76325548b 100644 --- a/app/controllers/api/v2/das_accounts_controller.rb +++ b/app/controllers/api/v2/das_accounts_controller.rb @@ -1,20 +1,25 @@ -module Api::V2 - class DasAccountsController < BaseController - def query - das = DasIndexerService.instance +module Api + module V2 + class DasAccountsController < BaseController + def query + das = DasIndexerService.instance - cache_keys = params[:addresses] + cache_keys = params[:addresses] - res = Rails.cache.read_multi(*cache_keys) - not_cached = cache_keys - res.keys - to_cache = {} - not_cached.each do |address| - name = das.reverse_record(address) - res[address] = name - to_cache[address] = name + res = Rails.cache.read_multi(*cache_keys) + not_cached = cache_keys - res.keys + to_cache = {} + not_cached.each do |address| + next unless QueryKeyUtils.valid_address?(address) + + name = das.reverse_record(address) + res[address] = name + to_cache[address] = name + end + Rails.cache.write_multi(to_cache, expires_in: 1.hour) + + render json: res end - Rails.cache.write_multi(to_cache, expires_in: 1.hour) - render json: res end end end diff --git a/test/controllers/api/v2/das_accounts_controller_test.rb b/test/controllers/api/v2/das_accounts_controller_test.rb index 543043a79..52e02f838 100644 --- a/test/controllers/api/v2/das_accounts_controller_test.rb +++ b/test/controllers/api/v2/das_accounts_controller_test.rb @@ -7,16 +7,20 @@ def after_setup super SecureRandom.stubs(:uuid).returns("11111111-1111-1111-1111-111111111111") end + + test "should return empty address alias for an invalid address" do + post api_v2_das_accounts_url, params: { addresses: ["test"] } + + assert_response :success + assert_equal json, {} + end + test "should return corresponding address alias" do DasIndexerService.any_instance.stubs(:reverse_record).returns("test") - - post api_v2_das_accounts_url, params: {addresses: ["test"]} - + post api_v2_das_accounts_url, params: { addresses: ["ckt1qyqrdsefa43s6m882pcj53m4gdnj4k440axqswmu83"] } + assert_response :success - data = JSON.parse response.body - assert_equal data["test"], "test" - # rescue => e - # puts e.backtrace.join("\n") + assert_equal json["ckt1qyqrdsefa43s6m882pcj53m4gdnj4k440axqswmu83"], "test" end end end From a6175e594e3f152f4d763c019bc84cb02aebd77d Mon Sep 17 00:00:00 2001 From: Rabbit Date: Thu, 30 Nov 2023 17:57:08 +0800 Subject: [PATCH 3/3] feat: aggregate market data indicators in JSON format (#1520) --- app/controllers/api/v1/market_data_controller.rb | 4 ++++ app/models/market_data.rb | 4 ++++ config/routes.rb | 2 +- test/controllers/api/v1/market_data_controller_test.rb | 6 +++--- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/app/controllers/api/v1/market_data_controller.rb b/app/controllers/api/v1/market_data_controller.rb index 4e7f491df..5b8a2cfd5 100644 --- a/app/controllers/api/v1/market_data_controller.rb +++ b/app/controllers/api/v1/market_data_controller.rb @@ -3,6 +3,10 @@ module V1 class MarketDataController < ApplicationController skip_before_action :check_header_info + def index + render json: MarketData.new.indicators_json + end + def show render json: MarketData.new(indicator: params[:id]).call end diff --git a/app/models/market_data.rb b/app/models/market_data.rb index 1fb1343c8..338b8bf68 100644 --- a/app/models/market_data.rb +++ b/app/models/market_data.rb @@ -23,6 +23,10 @@ def call send(indicator) end + def indicators_json + VALID_INDICATORS.index_with { |indicator| send(indicator) } + end + def ecosystem_locked if current_timestamp < first_released_timestamp_other ECOSYSTEM_QUOTA * 0.97 diff --git a/config/routes.rb b/config/routes.rb index 46f05ca0d..4a91a6226 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -53,7 +53,7 @@ resources :daily_statistics, only: :show resources :block_statistics, only: :show ## TODO: unused route resources :epoch_statistics, only: :show - resources :market_data, only: :show + resources :market_data, only: [:index, :show] resources :udts, only: %i(index show update) do collection do get :download_csv diff --git a/test/controllers/api/v1/market_data_controller_test.rb b/test/controllers/api/v1/market_data_controller_test.rb index c21284292..f4c353516 100644 --- a/test/controllers/api/v1/market_data_controller_test.rb +++ b/test/controllers/api/v1/market_data_controller_test.rb @@ -36,7 +36,7 @@ class MarketDataControllerTest < ActionDispatch::IntegrationTest create(:daily_statistic, treasury_amount: "45507635189304330.674891957030103511696912093394364431189654516859837775", created_at_unixtimestamp: Time.current.yesterday.beginning_of_day.to_i) end - test "should set right content type when call index" do + test "should set right content type when call show" do create(:address, address_hash: "ckb1qyqy6mtud5sgctjwgg6gydd0ea05mr339lnslczzrc", balance: 10**8 * 1000) valid_get api_v1_market_datum_url("circulating_supply") @@ -69,9 +69,9 @@ class MarketDataControllerTest < ActionDispatch::IntegrationTest test "should return current circulating supply" do create(:address, address_hash: "ckb1qyqy6mtud5sgctjwgg6gydd0ea05mr339lnslczzrc", balance: 10**8 * 1000) valid_get api_v1_market_datum_url("circulating_supply") - MarketData.new(indicator: "circulating_supply").call + result = MarketData.new(indicator: "circulating_supply").call - assert_equal MarketData.new(indicator: "circulating_supply").call.to_s, json + assert_equal result.to_s, json end end end