From 6a371fd257121e5f74db2dddab1a4e006a30de5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Fri, 11 Oct 2024 14:53:54 +0200 Subject: [PATCH 1/9] Rename to `node.backlog_scan` --- nano/core_test/backlog.cpp | 2 +- nano/core_test/optimistic_scheduler.cpp | 2 +- nano/node/json_handler.cpp | 2 +- nano/node/node.cpp | 8 ++++---- nano/node/node.hpp | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/nano/core_test/backlog.cpp b/nano/core_test/backlog.cpp index 21a2e80ebc..9de81919d0 100644 --- a/nano/core_test/backlog.cpp +++ b/nano/core_test/backlog.cpp @@ -22,7 +22,7 @@ TEST (backlog, population) nano::test::system system{}; auto & node = *system.add_node (); - node.backlog.activate_callback.add ([&] (nano::secure::transaction const & transaction, nano::account const & account) { + node.backlog_scan.activate_callback.add ([&] (nano::secure::transaction const & transaction, nano::account const & account) { nano::lock_guard lock{ mutex }; activated.insert (account); diff --git a/nano/core_test/optimistic_scheduler.cpp b/nano/core_test/optimistic_scheduler.cpp index 25d0771db3..d7d6abced3 100644 --- a/nano/core_test/optimistic_scheduler.cpp +++ b/nano/core_test/optimistic_scheduler.cpp @@ -98,7 +98,7 @@ TEST (optimistic_scheduler, under_gap_threshold) nano::test::confirm (node.ledger, blocks.at (55)); // Manually trigger backlog scan - node.backlog.trigger (); + node.backlog_scan.trigger (); // Ensure unconfirmed account head block gets activated auto const & block = blocks.back (); diff --git a/nano/node/json_handler.cpp b/nano/node/json_handler.cpp index 91dca52cd4..af496a963e 100644 --- a/nano/node/json_handler.cpp +++ b/nano/node/json_handler.cpp @@ -5154,7 +5154,7 @@ void nano::json_handler::work_peers_clear () void nano::json_handler::populate_backlog () { - node.backlog.trigger (); + node.backlog_scan.trigger (); response_l.put ("success", ""); response_errors (); } diff --git a/nano/node/node.cpp b/nano/node/node.cpp index 16d5a6f8c1..350b94a0e9 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -154,8 +154,8 @@ nano::node::node (std::shared_ptr io_ctx_a, std::filesy aggregator_impl{ std::make_unique (config.request_aggregator, *this, stats, generator, final_generator, history, ledger, wallets, vote_router) }, aggregator{ *aggregator_impl }, wallets (wallets_store.init_error (), *this), - backlog_impl{ std::make_unique (config.backlog_population, scheduler, ledger, stats) }, - backlog{ *backlog_impl }, + backlog_scan_impl{ std::make_unique (config.backlog_population, scheduler, ledger, stats) }, + backlog_scan{ *backlog_scan_impl }, bootstrap_server_impl{ std::make_unique (config.bootstrap_server, store, ledger, network_params.network, stats) }, bootstrap_server{ *bootstrap_server_impl }, bootstrap_impl{ std::make_unique (config, block_processor, ledger, network, stats, logger) }, @@ -633,7 +633,7 @@ void nano::node::start () confirming_set.start (); scheduler.start (); aggregator.start (); - backlog.start (); + backlog_scan.start (); bootstrap_server.start (); bootstrap.start (); websocket.start (); @@ -664,7 +664,7 @@ void nano::node::stop () // Cancels ongoing work generation tasks, which may be blocking other threads // No tasks may wait for work generation in I/O threads, or termination signal capturing will be unable to call node::stop() distributed_work.stop (); - backlog.stop (); + backlog_scan.stop (); bootstrap.stop (); rep_crawler.stop (); unchecked.stop (); diff --git a/nano/node/node.hpp b/nano/node/node.hpp index 2d4f4a351e..1912e70992 100644 --- a/nano/node/node.hpp +++ b/nano/node/node.hpp @@ -205,8 +205,8 @@ class node final : public std::enable_shared_from_this std::unique_ptr aggregator_impl; nano::request_aggregator & aggregator; nano::wallets wallets; - std::unique_ptr backlog_impl; - nano::backlog_population & backlog; + std::unique_ptr backlog_scan_impl; + nano::backlog_population & backlog_scan; std::unique_ptr bootstrap_server_impl; nano::bootstrap_server & bootstrap_server; std::unique_ptr bootstrap_impl; From 681d5f84e427fe538567b3fcad193e1df397d56d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Fri, 11 Oct 2024 15:00:02 +0200 Subject: [PATCH 2/9] Rename class to `backlog_scan` --- nano/node/CMakeLists.txt | 4 +-- ...acklog_population.cpp => backlog_scan.cpp} | 26 +++++++++---------- ...acklog_population.hpp => backlog_scan.hpp} | 20 +++++--------- nano/node/node.cpp | 4 +-- nano/node/node.hpp | 6 ++--- nano/node/nodeconfig.hpp | 4 +-- 6 files changed, 28 insertions(+), 36 deletions(-) rename nano/node/{backlog_population.cpp => backlog_scan.cpp} (81%) rename nano/node/{backlog_population.hpp => backlog_scan.hpp} (84%) diff --git a/nano/node/CMakeLists.txt b/nano/node/CMakeLists.txt index f50649b047..27b95f0d22 100644 --- a/nano/node/CMakeLists.txt +++ b/nano/node/CMakeLists.txt @@ -16,8 +16,8 @@ add_library( ${platform_sources} active_elections.hpp active_elections.cpp - backlog_population.hpp - backlog_population.cpp + backlog_scan.hpp + backlog_scan.cpp bandwidth_limiter.hpp bandwidth_limiter.cpp blockprocessor.hpp diff --git a/nano/node/backlog_population.cpp b/nano/node/backlog_scan.cpp similarity index 81% rename from nano/node/backlog_population.cpp rename to nano/node/backlog_scan.cpp index 284b7c599e..9d59a24e69 100644 --- a/nano/node/backlog_population.cpp +++ b/nano/node/backlog_scan.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #include #include #include @@ -8,7 +8,7 @@ #include #include -nano::backlog_population::backlog_population (backlog_population_config const & config_a, nano::scheduler::component & schedulers, nano::ledger & ledger, nano::stats & stats_a) : +nano::backlog_scan::backlog_scan (backlog_scan_config const & config_a, nano::scheduler::component & schedulers, nano::ledger & ledger, nano::stats & stats_a) : config{ config_a }, schedulers{ schedulers }, ledger{ ledger }, @@ -16,13 +16,13 @@ nano::backlog_population::backlog_population (backlog_population_config const & { } -nano::backlog_population::~backlog_population () +nano::backlog_scan::~backlog_scan () { // Thread must be stopped before destruction debug_assert (!thread.joinable ()); } -void nano::backlog_population::start () +void nano::backlog_scan::start () { debug_assert (!thread.joinable ()); @@ -32,7 +32,7 @@ void nano::backlog_population::start () } }; } -void nano::backlog_population::stop () +void nano::backlog_scan::stop () { { nano::lock_guard lock{ mutex }; @@ -42,7 +42,7 @@ void nano::backlog_population::stop () nano::join_or_pass (thread); } -void nano::backlog_population::trigger () +void nano::backlog_scan::trigger () { { nano::unique_lock lock{ mutex }; @@ -51,17 +51,17 @@ void nano::backlog_population::trigger () notify (); } -void nano::backlog_population::notify () +void nano::backlog_scan::notify () { condition.notify_all (); } -bool nano::backlog_population::predicate () const +bool nano::backlog_scan::predicate () const { return triggered || config.enable; } -void nano::backlog_population::run () +void nano::backlog_scan::run () { nano::unique_lock lock{ mutex }; while (!stopped) @@ -80,7 +80,7 @@ void nano::backlog_population::run () } } -void nano::backlog_population::populate_backlog (nano::unique_lock & lock) +void nano::backlog_scan::populate_backlog (nano::unique_lock & lock) { debug_assert (config.frequency > 0); @@ -125,7 +125,7 @@ void nano::backlog_population::populate_backlog (nano::unique_lock } } -void nano::backlog_population::activate (secure::transaction const & transaction, nano::account const & account, nano::account_info const & account_info) +void nano::backlog_scan::activate (secure::transaction const & transaction, nano::account const & account, nano::account_info const & account_info) { auto const maybe_conf_info = ledger.store.confirmation_height.get (transaction, account); auto const conf_info = maybe_conf_info.value_or (nano::confirmation_height_info{}); @@ -146,7 +146,7 @@ void nano::backlog_population::activate (secure::transaction const & transaction * backlog_population_config */ -nano::error nano::backlog_population_config::serialize (nano::tomlconfig & toml) const +nano::error nano::backlog_scan_config::serialize (nano::tomlconfig & toml) const { toml.put ("enable", enable, "Control if ongoing backlog population is enabled. If not, backlog population can still be triggered by RPC \ntype:bool"); toml.put ("batch_size", batch_size, "Number of accounts per second to process when doing backlog population scan. Increasing this value will help unconfirmed frontiers get into election prioritization queue faster, however it will also increase resource usage. \ntype:uint"); @@ -155,7 +155,7 @@ nano::error nano::backlog_population_config::serialize (nano::tomlconfig & toml) return toml.get_error (); } -nano::error nano::backlog_population_config::deserialize (nano::tomlconfig & toml) +nano::error nano::backlog_scan_config::deserialize (nano::tomlconfig & toml) { toml.get ("enable", enable); toml.get ("batch_size", batch_size); diff --git a/nano/node/backlog_population.hpp b/nano/node/backlog_scan.hpp similarity index 84% rename from nano/node/backlog_population.hpp rename to nano/node/backlog_scan.hpp index 37123f4ba9..f9b2ba8aad 100644 --- a/nano/node/backlog_population.hpp +++ b/nano/node/backlog_scan.hpp @@ -3,24 +3,16 @@ #include #include #include +#include #include #include #include #include -namespace nano::secure -{ -class transaction; -} namespace nano { -class account_info; -class election_scheduler; -class ledger; -class stats; - -class backlog_population_config final +class backlog_scan_config final { public: nano::error deserialize (nano::tomlconfig &); @@ -35,11 +27,11 @@ class backlog_population_config final unsigned frequency{ 10 }; }; -class backlog_population final +class backlog_scan final { public: - backlog_population (backlog_population_config const &, nano::scheduler::component &, nano::ledger &, nano::stats &); - ~backlog_population (); + backlog_scan (backlog_scan_config const &, nano::scheduler::component &, nano::ledger &, nano::stats &); + ~backlog_scan (); void start (); void stop (); @@ -58,7 +50,7 @@ class backlog_population final callback_t activate_callback; private: // Dependencies - backlog_population_config const & config; + backlog_scan_config const & config; nano::scheduler::component & schedulers; nano::ledger & ledger; nano::stats & stats; diff --git a/nano/node/node.cpp b/nano/node/node.cpp index 350b94a0e9..45e73b1692 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include #include #include @@ -154,7 +154,7 @@ nano::node::node (std::shared_ptr io_ctx_a, std::filesy aggregator_impl{ std::make_unique (config.request_aggregator, *this, stats, generator, final_generator, history, ledger, wallets, vote_router) }, aggregator{ *aggregator_impl }, wallets (wallets_store.init_error (), *this), - backlog_scan_impl{ std::make_unique (config.backlog_population, scheduler, ledger, stats) }, + backlog_scan_impl{ std::make_unique (config.backlog_population, scheduler, ledger, stats) }, backlog_scan{ *backlog_scan_impl }, bootstrap_server_impl{ std::make_unique (config.bootstrap_server, store, ledger, network_params.network, stats) }, bootstrap_server{ *bootstrap_server_impl }, diff --git a/nano/node/node.hpp b/nano/node/node.hpp index 1912e70992..a471942378 100644 --- a/nano/node/node.hpp +++ b/nano/node/node.hpp @@ -33,7 +33,7 @@ namespace nano { class active_elections; -class backlog_population; +class backlog_scan; class bandwidth_limiter; class confirming_set; class message_processor; @@ -205,8 +205,8 @@ class node final : public std::enable_shared_from_this std::unique_ptr aggregator_impl; nano::request_aggregator & aggregator; nano::wallets wallets; - std::unique_ptr backlog_scan_impl; - nano::backlog_population & backlog_scan; + std::unique_ptr backlog_scan_impl; + nano::backlog_scan & backlog_scan; std::unique_ptr bootstrap_server_impl; nano::bootstrap_server & bootstrap_server; std::unique_ptr bootstrap_impl; diff --git a/nano/node/nodeconfig.hpp b/nano/node/nodeconfig.hpp index aa5a13f8f4..15498843f8 100644 --- a/nano/node/nodeconfig.hpp +++ b/nano/node/nodeconfig.hpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include #include @@ -146,7 +146,7 @@ class node_config nano::local_block_broadcaster_config local_block_broadcaster; nano::confirming_set_config confirming_set; nano::monitor_config monitor; - nano::backlog_population_config backlog_population; + nano::backlog_scan_config backlog_population; public: /** Entry is ignored if it cannot be parsed as a valid address:port */ From 83954e40ec9817fbca6dade1783e01e5335d7aeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Fri, 11 Oct 2024 15:08:36 +0200 Subject: [PATCH 3/9] Rename config to `backlog_scan` --- nano/core_test/active_elections.cpp | 24 +++++++-------- nano/core_test/bootstrap.cpp | 6 ++-- nano/core_test/confirming_set.cpp | 6 ++-- nano/core_test/election.cpp | 10 +++---- nano/core_test/election_scheduler.cpp | 2 +- nano/core_test/ledger.cpp | 4 +-- nano/core_test/ledger_confirm.cpp | 8 ++--- nano/core_test/node.cpp | 40 ++++++++++++------------- nano/core_test/optimistic_scheduler.cpp | 2 +- nano/core_test/request_aggregator.cpp | 14 ++++----- nano/core_test/toml.cpp | 16 +++++----- nano/core_test/vote_processor.cpp | 14 ++++----- nano/core_test/voting.cpp | 4 +-- nano/core_test/wallet.cpp | 2 +- nano/core_test/wallets.cpp | 2 +- nano/lib/thread_roles.cpp | 4 +-- nano/lib/thread_roles.hpp | 2 +- nano/node/backlog_scan.cpp | 4 +-- nano/node/node.cpp | 2 +- nano/node/nodeconfig.cpp | 12 ++++---- nano/node/nodeconfig.hpp | 2 +- nano/rpc_test/receivable.cpp | 4 +-- nano/rpc_test/rpc.cpp | 8 ++--- nano/slow_test/node.cpp | 18 +++++------ nano/slow_test/vote_cache.cpp | 4 +-- 25 files changed, 107 insertions(+), 107 deletions(-) diff --git a/nano/core_test/active_elections.cpp b/nano/core_test/active_elections.cpp index 32d2a89641..52f409ccc0 100644 --- a/nano/core_test/active_elections.cpp +++ b/nano/core_test/active_elections.cpp @@ -170,7 +170,7 @@ TEST (active_elections, DISABLED_keep_local) // Bound to 2, won't drop wallet created transactions, but good to test dropping remote node_config.active_elections.size = 2; // Disable frontier confirmation to allow the test to finish before - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node = *system.add_node (node_config); auto & wallet (*system.wallet (0)); @@ -330,7 +330,7 @@ TEST (inactive_votes_cache, existing_vote) { nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node = *system.add_node (node_config); nano::block_hash latest (node.latest (nano::dev::genesis_key.pub)); nano::keypair key; @@ -384,7 +384,7 @@ TEST (inactive_votes_cache, multiple_votes) { nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node = *system.add_node (node_config); nano::keypair key1; nano::block_builder builder; @@ -437,7 +437,7 @@ TEST (inactive_votes_cache, election_start) { nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; node_config.priority_scheduler.enable = false; node_config.optimistic_scheduler.enable = false; auto & node = *system.add_node (node_config); @@ -543,7 +543,7 @@ TEST (active_elections, vote_replays) nano::test::system system; nano::node_config node_config = system.default_config (); node_config.enable_voting = false; - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node = *system.add_node (node_config); nano::keypair key; nano::state_block_builder builder; @@ -699,7 +699,7 @@ TEST (active_elections, republish_winner) { nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node1 = *system.add_node (node_config); node_config.peering_port = system.get_available_port (); auto & node2 = *system.add_node (node_config); @@ -765,7 +765,7 @@ TEST (active_elections, fork_filter_cleanup) nano::test::system system{}; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node1 = *system.add_node (node_config); nano::keypair key{}; @@ -846,7 +846,7 @@ TEST (active_elections, fork_replacement_tally) { nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node1 (*system.add_node (node_config)); size_t const reps_count = 20; @@ -1003,7 +1003,7 @@ TEST (active_elections, confirmation_consistency) { nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node = *system.add_node (node_config); system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); for (unsigned i = 0; i < 10; ++i) @@ -1095,7 +1095,7 @@ TEST (active_elections, activate_account_chain) nano::test::system system; nano::node_flags flags; nano::node_config config = system.default_config (); - config.backlog_population.enable = false; + config.backlog_scan.enable = false; auto & node = *system.add_node (config, flags); nano::keypair key; @@ -1187,7 +1187,7 @@ TEST (active_elections, activate_inactive) nano::test::system system; nano::node_flags flags; nano::node_config config = system.default_config (); - config.backlog_population.enable = false; + config.backlog_scan.enable = false; auto & node = *system.add_node (config, flags); nano::keypair key; @@ -1337,7 +1337,7 @@ TEST (active_elections, limit_vote_hinted_elections) nano::test::system system; nano::node_config config = system.default_config (); const int aec_limit = 10; - config.backlog_population.enable = false; + config.backlog_scan.enable = false; config.optimistic_scheduler.enable = false; config.active_elections.size = aec_limit; config.active_elections.hinted_limit_percentage = 10; // Should give us a limit of 1 hinted election diff --git a/nano/core_test/bootstrap.cpp b/nano/core_test/bootstrap.cpp index 1073a83a39..ec30708ebf 100644 --- a/nano/core_test/bootstrap.cpp +++ b/nano/core_test/bootstrap.cpp @@ -299,7 +299,7 @@ TEST (bootstrap, frontier_scan) config.bootstrap.enable_scan = false; config.bootstrap.enable_dependency_walker = false; // Disable election activation - config.backlog_population.enable = false; + config.backlog_scan.enable = false; config.priority_scheduler.enable = false; config.optimistic_scheduler.enable = false; config.hinted_scheduler.enable = false; @@ -395,7 +395,7 @@ TEST (bootstrap, frontier_scan_pending) config.bootstrap.enable_scan = false; config.bootstrap.enable_dependency_walker = false; // Disable election activation - config.backlog_population.enable = false; + config.backlog_scan.enable = false; config.priority_scheduler.enable = false; config.optimistic_scheduler.enable = false; config.hinted_scheduler.enable = false; @@ -477,7 +477,7 @@ TEST (bootstrap, frontier_scan_cannot_prioritize) config.bootstrap.enable_scan = false; config.bootstrap.enable_dependency_walker = false; // Disable election activation - config.backlog_population.enable = false; + config.backlog_scan.enable = false; config.priority_scheduler.enable = false; config.optimistic_scheduler.enable = false; config.hinted_scheduler.enable = false; diff --git a/nano/core_test/confirming_set.cpp b/nano/core_test/confirming_set.cpp index f412b0930a..0b891ee0dc 100644 --- a/nano/core_test/confirming_set.cpp +++ b/nano/core_test/confirming_set.cpp @@ -100,7 +100,7 @@ TEST (confirmation_callback, observer_callbacks) nano::test::system system; nano::node_flags node_flags; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto node = system.add_node (node_config, node_flags); system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); @@ -145,7 +145,7 @@ TEST (confirmation_callback, confirmed_history) { nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; node_config.bootstrap.enable = false; auto node = system.add_node (node_config); @@ -217,7 +217,7 @@ TEST (confirmation_callback, dependent_election) nano::test::system system; nano::node_flags node_flags; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto node = system.add_node (node_config, node_flags); nano::block_hash latest (node->latest (nano::dev::genesis_key.pub)); diff --git a/nano/core_test/election.cpp b/nano/core_test/election.cpp index 200f84ffbb..6c386ce085 100644 --- a/nano/core_test/election.cpp +++ b/nano/core_test/election.cpp @@ -36,7 +36,7 @@ TEST (election, quorum_minimum_flip_success) nano::node_config node_config = system.default_config (); node_config.online_weight_minimum = nano::dev::constants.genesis_amount; - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node1 = *system.add_node (node_config); auto const latest_hash = nano::dev::genesis->hash (); @@ -86,7 +86,7 @@ TEST (election, quorum_minimum_flip_fail) nano::test::system system; nano::node_config node_config = system.default_config (); node_config.online_weight_minimum = nano::dev::constants.genesis_amount; - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node = *system.add_node (node_config); nano::state_block_builder builder; @@ -137,7 +137,7 @@ TEST (election, quorum_minimum_confirm_success) nano::test::system system; nano::node_config node_config = system.default_config (); node_config.online_weight_minimum = nano::dev::constants.genesis_amount; - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node1 = *system.add_node (node_config); nano::keypair key1; nano::block_builder builder; @@ -167,7 +167,7 @@ TEST (election, quorum_minimum_confirm_fail) nano::test::system system; nano::node_config node_config = system.default_config (); node_config.online_weight_minimum = nano::dev::constants.genesis_amount; - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node1 = *system.add_node (node_config); nano::block_builder builder; @@ -205,7 +205,7 @@ TEST (election, quorum_minimum_update_weight_before_quorum_checks) nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node1 = *system.add_node (node_config); system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); diff --git a/nano/core_test/election_scheduler.cpp b/nano/core_test/election_scheduler.cpp index 422df99719..12e3cc2901 100644 --- a/nano/core_test/election_scheduler.cpp +++ b/nano/core_test/election_scheduler.cpp @@ -178,7 +178,7 @@ TEST (election_scheduler, no_vacancy) nano::node_config config = system.default_config (); config.active_elections.size = 1; - config.backlog_population.enable = false; + config.backlog_scan.enable = false; auto & node = *system.add_node (config); nano::state_block_builder builder{}; diff --git a/nano/core_test/ledger.cpp b/nano/core_test/ledger.cpp index 32658ba76c..a91949df60 100644 --- a/nano/core_test/ledger.cpp +++ b/nano/core_test/ledger.cpp @@ -1018,7 +1018,7 @@ TEST (votes, add_existing) nano::test::system system; nano::node_config node_config = system.default_config (); node_config.online_weight_minimum = nano::dev::constants.genesis_amount; - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node1 = *system.add_node (node_config); nano::keypair key1; nano::block_builder builder; @@ -4267,7 +4267,7 @@ TEST (ledger, unchecked_epoch_invalid) { nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node1 (*system.add_node (node_config)); nano::keypair destination; nano::block_builder builder; diff --git a/nano/core_test/ledger_confirm.cpp b/nano/core_test/ledger_confirm.cpp index f9bfeff998..a4789f296c 100644 --- a/nano/core_test/ledger_confirm.cpp +++ b/nano/core_test/ledger_confirm.cpp @@ -58,7 +58,7 @@ TEST (ledger_confirm, multiple_accounts) nano::test::system system; nano::node_flags node_flags; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto node = system.add_node (node_config, node_flags); nano::keypair key1; nano::keypair key2; @@ -232,7 +232,7 @@ TEST (ledger_confirm, send_receive_between_2_accounts) nano::test::system system; nano::node_flags node_flags; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto node = system.add_node (node_config, node_flags); nano::keypair key1; nano::block_hash latest (node->latest (nano::dev::genesis_key.pub)); @@ -361,7 +361,7 @@ TEST (ledger_confirm, send_receive_self) nano::test::system system; nano::node_flags node_flags; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto node = system.add_node (node_config, node_flags); nano::block_hash latest (node->latest (nano::dev::genesis_key.pub)); @@ -449,7 +449,7 @@ TEST (ledger_confirm, all_block_types) nano::test::system system; nano::node_flags node_flags; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto node = system.add_node (node_config, node_flags); nano::block_hash latest (node->latest (nano::dev::genesis_key.pub)); nano::keypair key1; diff --git a/nano/core_test/node.cpp b/nano/core_test/node.cpp index 2db65d6afc..1d9118ec2c 100644 --- a/nano/core_test/node.cpp +++ b/nano/core_test/node.cpp @@ -263,7 +263,7 @@ TEST (node, auto_bootstrap) { nano::test::system system; nano::node_config config (system.get_available_port ()); - config.backlog_population.enable = false; + config.backlog_scan.enable = false; nano::node_flags node_flags; node_flags.disable_bootstrap_bulk_push_client = true; node_flags.disable_lazy_bootstrap = true; @@ -291,7 +291,7 @@ TEST (node, auto_bootstrap_reverse) { nano::test::system system; nano::node_config config (system.get_available_port ()); - config.backlog_population.enable = false; + config.backlog_scan.enable = false; nano::node_flags node_flags; node_flags.disable_bootstrap_bulk_push_client = true; node_flags.disable_lazy_bootstrap = true; @@ -364,7 +364,7 @@ TEST (node, search_receivable_confirmed) { nano::test::system system; nano::node_config node_config (system.get_available_port ()); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto node = system.add_node (node_config); nano::keypair key2; system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); @@ -393,7 +393,7 @@ TEST (node, search_receivable_pruned) { nano::test::system system; nano::node_config node_config (system.get_available_port ()); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto node1 = system.add_node (node_config); nano::node_flags node_flags; node_flags.enable_pruning = true; @@ -699,7 +699,7 @@ TEST (node, fork_multi_flip) nano::test::system system; nano::node_flags node_flags; nano::node_config node_config (system.get_available_port ()); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node1 (*system.add_node (node_config, node_flags, type)); node_config.peering_port = system.get_available_port (); node_config.bootstrap.account_sets.cooldown = 100ms; // Reduce cooldown to speed up fork resolution @@ -752,7 +752,7 @@ TEST (node, fork_bootstrap_flip) { nano::test::system system; nano::node_config config1{ system.get_available_port () }; - config1.backlog_population.enable = false; + config1.backlog_scan.enable = false; nano::node_flags node_flags; node_flags.disable_bootstrap_bulk_push_client = true; node_flags.disable_lazy_bootstrap = true; @@ -1159,7 +1159,7 @@ TEST (node, DISABLED_broadcast_elected) nano::node_flags node_flags; nano::test::system system; nano::node_config node_config (system.get_available_port ()); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto node0 = system.add_node (node_config, node_flags, type); node_config.peering_port = system.get_available_port (); auto node1 = system.add_node (node_config, node_flags, type); @@ -1286,7 +1286,7 @@ TEST (node, rep_self_vote) nano::test::system system; nano::node_config node_config (system.get_available_port ()); node_config.online_weight_minimum = std::numeric_limits::max (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto node0 = system.add_node (node_config); nano::keypair rep_big; nano::block_builder builder; @@ -1689,7 +1689,7 @@ TEST (node, DISABLED_local_votes_cache) { nano::test::system system; nano::node_config node_config (system.get_available_port ()); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; node_config.receive_minimum = nano::dev::constants.genesis_amount; auto & node (*system.add_node (node_config)); nano::state_block_builder builder; @@ -1773,7 +1773,7 @@ TEST (node, DISABLED_local_votes_cache_batch) { nano::test::system system; nano::node_config node_config (system.get_available_port ()); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node (*system.add_node (node_config)); ASSERT_GE (node.network_params.voting.max_cache, 2); system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); @@ -1847,7 +1847,7 @@ TEST (node, DISABLED_local_votes_cache_generate_new_vote) { nano::test::system system; nano::node_config node_config (system.get_available_port ()); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node (*system.add_node (node_config)); system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); @@ -1901,7 +1901,7 @@ TEST (node, DISABLED_local_votes_cache_fork) node_flags.disable_legacy_bootstrap = true; node_flags.disable_wallet_bootstrap = true; nano::node_config node_config (system.get_available_port ()); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node1 (*system.add_node (node_config, node_flags)); system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); auto send1 = nano::state_block_builder () @@ -2150,7 +2150,7 @@ TEST (node, epoch_conflict_confirm) { nano::test::system system; nano::node_config node_config (system.get_available_port ()); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node0 = *system.add_node (node_config); node_config.peering_port = system.get_available_port (); auto & node1 = *system.add_node (node_config); @@ -2661,7 +2661,7 @@ TEST (node, bidirectional_tcp) node_flags.disable_lazy_bootstrap = true; node_flags.disable_wallet_bootstrap = true; nano::node_config node_config (system.get_available_port ()); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto node1 = system.add_node (node_config, node_flags); node_config.peering_port = system.get_available_port (); node_config.tcp_incoming_connections_max = 0; // Disable incoming TCP connections for node 2 @@ -2854,7 +2854,7 @@ TEST (node, rollback_gap_source) { nano::test::system system; nano::node_config node_config (system.get_available_port ()); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node = *system.add_node (node_config); nano::state_block_builder builder; nano::keypair key; @@ -2922,7 +2922,7 @@ TEST (node, dependency_graph) { nano::test::system system; nano::node_config config (system.get_available_port ()); - config.backlog_population.enable = false; + config.backlog_scan.enable = false; auto & node = *system.add_node (config); nano::state_block_builder builder; @@ -3120,10 +3120,10 @@ TEST (node, dependency_graph_frontier) { nano::test::system system; nano::node_config config (system.get_available_port ()); - config.backlog_population.enable = false; + config.backlog_scan.enable = false; auto & node1 = *system.add_node (config); config.peering_port = system.get_available_port (); - config.backlog_population.enable = true; + config.backlog_scan.enable = true; auto & node2 = *system.add_node (config); nano::state_block_builder builder; @@ -3287,9 +3287,9 @@ TEST (node, deferred_dependent_elections) { nano::test::system system; nano::node_config node_config_1{ system.get_available_port () }; - node_config_1.backlog_population.enable = false; + node_config_1.backlog_scan.enable = false; nano::node_config node_config_2{ system.get_available_port () }; - node_config_2.backlog_population.enable = false; + node_config_2.backlog_scan.enable = false; nano::node_flags flags; flags.disable_request_loop = true; auto & node = *system.add_node (node_config_1, flags); diff --git a/nano/core_test/optimistic_scheduler.cpp b/nano/core_test/optimistic_scheduler.cpp index d7d6abced3..fa2107d789 100644 --- a/nano/core_test/optimistic_scheduler.cpp +++ b/nano/core_test/optimistic_scheduler.cpp @@ -85,7 +85,7 @@ TEST (optimistic_scheduler, under_gap_threshold) { nano::test::system system{}; nano::node_config config = system.default_config (); - config.backlog_population.enable = false; + config.backlog_scan.enable = false; auto & node = *system.add_node (config); // Must be smaller than optimistic scheduler `gap_threshold` diff --git a/nano/core_test/request_aggregator.cpp b/nano/core_test/request_aggregator.cpp index 5e99210c28..33b96df5fc 100644 --- a/nano/core_test/request_aggregator.cpp +++ b/nano/core_test/request_aggregator.cpp @@ -21,7 +21,7 @@ TEST (request_aggregator, one) { nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node (*system.add_node (node_config)); system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); nano::block_builder builder; @@ -71,7 +71,7 @@ TEST (request_aggregator, one_update) { nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node (*system.add_node (node_config)); system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); nano::keypair key1; @@ -137,7 +137,7 @@ TEST (request_aggregator, two) { nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node (*system.add_node (node_config)); system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); nano::keypair key1; @@ -208,7 +208,7 @@ TEST (request_aggregator, two_endpoints) { nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; nano::node_flags node_flags; node_flags.disable_rep_crawler = true; auto & node1 (*system.add_node (node_config, node_flags)); @@ -266,7 +266,7 @@ TEST (request_aggregator, split) size_t max_vbh = nano::network::confirm_ack_hashes_max; nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node (*system.add_node (node_config)); system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); std::vector> request; @@ -319,7 +319,7 @@ TEST (request_aggregator, channel_max_queue) { nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; node_config.request_aggregator.max_queue = 0; auto & node (*system.add_node (node_config)); system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); @@ -349,7 +349,7 @@ TEST (request_aggregator, DISABLED_unique) { nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node (*system.add_node (node_config)); system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); nano::block_builder builder; diff --git a/nano/core_test/toml.cpp b/nano/core_test/toml.cpp index 0da8a59f7b..1cbca2edc5 100644 --- a/nano/core_test/toml.cpp +++ b/nano/core_test/toml.cpp @@ -116,7 +116,7 @@ TEST (toml, daemon_config_deserialize_defaults) std::stringstream ss; ss << R"toml( [node] - [node.backlog_population] + [node.backlog_scan] [node.bootstrap] [node.bootstrap_server] [node.block_processor] @@ -197,9 +197,9 @@ TEST (toml, daemon_config_deserialize_defaults) ASSERT_EQ (conf.node.max_queued_requests, defaults.node.max_queued_requests); ASSERT_EQ (conf.node.request_aggregator_threads, defaults.node.request_aggregator_threads); ASSERT_EQ (conf.node.max_unchecked_blocks, defaults.node.max_unchecked_blocks); - ASSERT_EQ (conf.node.backlog_population.enable, defaults.node.backlog_population.enable); - ASSERT_EQ (conf.node.backlog_population.batch_size, defaults.node.backlog_population.batch_size); - ASSERT_EQ (conf.node.backlog_population.frequency, defaults.node.backlog_population.frequency); + ASSERT_EQ (conf.node.backlog_scan.enable, defaults.node.backlog_scan.enable); + ASSERT_EQ (conf.node.backlog_scan.batch_size, defaults.node.backlog_scan.batch_size); + ASSERT_EQ (conf.node.backlog_scan.frequency, defaults.node.backlog_scan.frequency); ASSERT_EQ (conf.node.enable_upnp, defaults.node.enable_upnp); ASSERT_EQ (conf.node.websocket_config.enabled, defaults.node.websocket_config.enabled); @@ -465,7 +465,7 @@ TEST (toml, daemon_config_deserialize_no_defaults) frontiers_confirmation = "always" enable_upnp = false - [node.backlog_population] + [node.backlog_scan] enable = false batch_size = 999 frequency = 999 @@ -704,9 +704,9 @@ TEST (toml, daemon_config_deserialize_no_defaults) ASSERT_NE (conf.node.work_threads, defaults.node.work_threads); ASSERT_NE (conf.node.max_queued_requests, defaults.node.max_queued_requests); ASSERT_NE (conf.node.request_aggregator_threads, defaults.node.request_aggregator_threads); - ASSERT_NE (conf.node.backlog_population.enable, defaults.node.backlog_population.enable); - ASSERT_NE (conf.node.backlog_population.batch_size, defaults.node.backlog_population.batch_size); - ASSERT_NE (conf.node.backlog_population.frequency, defaults.node.backlog_population.frequency); + ASSERT_NE (conf.node.backlog_scan.enable, defaults.node.backlog_scan.enable); + ASSERT_NE (conf.node.backlog_scan.batch_size, defaults.node.backlog_scan.batch_size); + ASSERT_NE (conf.node.backlog_scan.frequency, defaults.node.backlog_scan.frequency); ASSERT_NE (conf.node.enable_upnp, defaults.node.enable_upnp); ASSERT_NE (conf.node.websocket_config.enabled, defaults.node.websocket_config.enabled); diff --git a/nano/core_test/vote_processor.cpp b/nano/core_test/vote_processor.cpp index 4e237b34a7..ea9ada7809 100644 --- a/nano/core_test/vote_processor.cpp +++ b/nano/core_test/vote_processor.cpp @@ -21,7 +21,7 @@ TEST (vote_processor, codes) nano::test::system system; auto node_config = system.default_config (); // Disable all election schedulers - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; node_config.hinted_scheduler.enable = false; node_config.optimistic_scheduler.enable = false; auto & node = *system.add_node (node_config); @@ -159,10 +159,10 @@ TEST (vote_processor, no_broadcast_local) flags.disable_request_loop = true; nano::node_config config1, config2; config1.representative_vote_weight_minimum = 0; - config1.backlog_population.enable = false; + config1.backlog_scan.enable = false; auto & node (*system.add_node (config1, flags)); config2.representative_vote_weight_minimum = 0; - config2.backlog_population.enable = false; + config2.backlog_scan.enable = false; config2.peering_port = system.get_available_port (); system.add_node (config2, flags); nano::block_builder builder; @@ -214,10 +214,10 @@ TEST (vote_processor, local_broadcast_without_a_representative) flags.disable_request_loop = true; nano::node_config config1, config2; config1.representative_vote_weight_minimum = 0; - config1.backlog_population.enable = false; + config1.backlog_scan.enable = false; auto & node (*system.add_node (config1, flags)); config2.representative_vote_weight_minimum = 0; - config2.backlog_population.enable = false; + config2.backlog_scan.enable = false; config2.peering_port = system.get_available_port (); system.add_node (config2, flags); nano::block_builder builder; @@ -263,9 +263,9 @@ TEST (vote_processor, no_broadcast_local_with_a_principal_representative) nano::node_flags flags; flags.disable_request_loop = true; nano::node_config config1, config2; - config1.backlog_population.enable = false; + config1.backlog_scan.enable = false; auto & node (*system.add_node (config1, flags)); - config2.backlog_population.enable = false; + config2.backlog_scan.enable = false; config2.peering_port = system.get_available_port (); system.add_node (config2, flags); nano::block_builder builder; diff --git a/nano/core_test/voting.cpp b/nano/core_test/voting.cpp index 55d63259fc..dd0487ddfc 100644 --- a/nano/core_test/voting.cpp +++ b/nano/core_test/voting.cpp @@ -146,7 +146,7 @@ TEST (vote_spacing, prune) TEST (vote_spacing, vote_generator) { nano::node_config config; - config.backlog_population.enable = false; + config.backlog_scan.enable = false; config.active_elections.hinted_limit_percentage = 0; // Disable election hinting nano::test::system system; nano::node_flags node_flags; @@ -190,7 +190,7 @@ TEST (vote_spacing, vote_generator) TEST (vote_spacing, rapid) { nano::node_config config; - config.backlog_population.enable = false; + config.backlog_scan.enable = false; config.active_elections.hinted_limit_percentage = 0; // Disable election hinting nano::test::system system; nano::node_flags node_flags; diff --git a/nano/core_test/wallet.cpp b/nano/core_test/wallet.cpp index 79068966e8..6a95ba34ec 100644 --- a/nano/core_test/wallet.cpp +++ b/nano/core_test/wallet.cpp @@ -1150,7 +1150,7 @@ TEST (wallet, search_receivable) nano::test::system system; nano::node_config config = system.default_config (); config.enable_voting = false; - config.backlog_population.enable = false; + config.backlog_scan.enable = false; nano::node_flags flags; flags.disable_search_pending = true; auto & node (*system.add_node (config, flags)); diff --git a/nano/core_test/wallets.cpp b/nano/core_test/wallets.cpp index 71caf12ad1..4b6e2a09f0 100644 --- a/nano/core_test/wallets.cpp +++ b/nano/core_test/wallets.cpp @@ -195,7 +195,7 @@ TEST (wallets, search_receivable) nano::test::system system; nano::node_config config = system.default_config (); config.enable_voting = false; - config.backlog_population.enable = false; + config.backlog_scan.enable = false; nano::node_flags flags; flags.disable_search_pending = true; auto & node (*system.add_node (config, flags)); diff --git a/nano/lib/thread_roles.cpp b/nano/lib/thread_roles.cpp index efc6aa232d..89b2c5da0c 100644 --- a/nano/lib/thread_roles.cpp +++ b/nano/lib/thread_roles.cpp @@ -94,8 +94,8 @@ std::string nano::thread_role::get_string (nano::thread_role::name role) case nano::thread_role::name::unchecked: thread_role_name_string = "Unchecked"; break; - case nano::thread_role::name::backlog_population: - thread_role_name_string = "Backlog"; + case nano::thread_role::name::backlog_scan: + thread_role_name_string = "Backlog scan"; break; case nano::thread_role::name::vote_generator_queue: thread_role_name_string = "Voting que"; diff --git a/nano/lib/thread_roles.hpp b/nano/lib/thread_roles.hpp index 04325743e9..15b4a511cb 100644 --- a/nano/lib/thread_roles.hpp +++ b/nano/lib/thread_roles.hpp @@ -36,7 +36,7 @@ enum class name epoch_upgrader, db_parallel_traversal, unchecked, - backlog_population, + backlog_scan, vote_generator_queue, telemetry, bootstrap, diff --git a/nano/node/backlog_scan.cpp b/nano/node/backlog_scan.cpp index 9d59a24e69..8ba28a0659 100644 --- a/nano/node/backlog_scan.cpp +++ b/nano/node/backlog_scan.cpp @@ -27,7 +27,7 @@ void nano::backlog_scan::start () debug_assert (!thread.joinable ()); thread = std::thread{ [this] () { - nano::thread_role::set (nano::thread_role::name::backlog_population); + nano::thread_role::set (nano::thread_role::name::backlog_scan); run (); } }; } @@ -143,7 +143,7 @@ void nano::backlog_scan::activate (secure::transaction const & transaction, nano } /* - * backlog_population_config + * backlog_scan_config */ nano::error nano::backlog_scan_config::serialize (nano::tomlconfig & toml) const diff --git a/nano/node/node.cpp b/nano/node/node.cpp index 45e73b1692..6fd309447b 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -154,7 +154,7 @@ nano::node::node (std::shared_ptr io_ctx_a, std::filesy aggregator_impl{ std::make_unique (config.request_aggregator, *this, stats, generator, final_generator, history, ledger, wallets, vote_router) }, aggregator{ *aggregator_impl }, wallets (wallets_store.init_error (), *this), - backlog_scan_impl{ std::make_unique (config.backlog_population, scheduler, ledger, stats) }, + backlog_scan_impl{ std::make_unique (config.backlog_scan, scheduler, ledger, stats) }, backlog_scan{ *backlog_scan_impl }, bootstrap_server_impl{ std::make_unique (config.bootstrap_server, store, ledger, network_params.network, stats) }, bootstrap_server{ *bootstrap_server_impl }, diff --git a/nano/node/nodeconfig.cpp b/nano/node/nodeconfig.cpp index c7735299e2..8fb095afd8 100644 --- a/nano/node/nodeconfig.cpp +++ b/nano/node/nodeconfig.cpp @@ -258,9 +258,9 @@ nano::error nano::node_config::serialize_toml (nano::tomlconfig & toml) const monitor.serialize (monitor_l); toml.put_child ("monitor", monitor_l); - nano::tomlconfig backlog_population_l; - backlog_population.serialize (backlog_population_l); - toml.put_child ("backlog_population", backlog_population_l); + nano::tomlconfig backlog_scan_l; + backlog_scan.serialize (backlog_scan_l); + toml.put_child ("backlog_scan", backlog_scan_l); return toml.get_error (); } @@ -395,10 +395,10 @@ nano::error nano::node_config::deserialize_toml (nano::tomlconfig & toml) monitor.deserialize (config_l); } - if (toml.has_key ("backlog_population")) + if (toml.has_key ("backlog_scan")) { - auto config_l = toml.get_required_child ("backlog_population"); - backlog_population.deserialize (config_l); + auto config_l = toml.get_required_child ("backlog_scan"); + backlog_scan.deserialize (config_l); } /* diff --git a/nano/node/nodeconfig.hpp b/nano/node/nodeconfig.hpp index 15498843f8..ed4ca6fa3e 100644 --- a/nano/node/nodeconfig.hpp +++ b/nano/node/nodeconfig.hpp @@ -146,7 +146,7 @@ class node_config nano::local_block_broadcaster_config local_block_broadcaster; nano::confirming_set_config confirming_set; nano::monitor_config monitor; - nano::backlog_scan_config backlog_population; + nano::backlog_scan_config backlog_scan; public: /** Entry is ignored if it cannot be parsed as a valid address:port */ diff --git a/nano/rpc_test/receivable.cpp b/nano/rpc_test/receivable.cpp index 9199a14619..672a8f2584 100644 --- a/nano/rpc_test/receivable.cpp +++ b/nano/rpc_test/receivable.cpp @@ -133,7 +133,7 @@ TEST (rpc, receivable_unconfirmed) { nano::test::system system; nano::node_config config; - config.backlog_population.enable = false; + config.backlog_scan.enable = false; auto node = add_ipc_enabled_node (system, config); auto chain = nano::test::setup_chain (system, *node, 1, nano::dev::genesis_key, false); auto block1 = chain[0]; @@ -530,7 +530,7 @@ TEST (rpc, accounts_receivable_confirmed) { nano::test::system system; nano::node_config config; - config.backlog_population.enable = false; + config.backlog_scan.enable = false; auto node = add_ipc_enabled_node (system, config); auto chain = nano::test::setup_chain (system, *node, 1, nano::dev::genesis_key, false); auto block1 = chain[0]; diff --git a/nano/rpc_test/rpc.cpp b/nano/rpc_test/rpc.cpp index cfc63f0b3c..4969605f7d 100644 --- a/nano/rpc_test/rpc.cpp +++ b/nano/rpc_test/rpc.cpp @@ -2924,7 +2924,7 @@ TEST (rpc, accounts_balances_unopened_account_with_receivables) { nano::test::system system; nano::node_config config; - config.backlog_population.enable = false; + config.backlog_scan.enable = false; auto node = add_ipc_enabled_node (system, config); // send a 1 raw to the unopened account which will have receivables @@ -3228,7 +3228,7 @@ TEST (rpc, pending_exists) { nano::test::system system; nano::node_config config; - config.backlog_population.enable = false; + config.backlog_scan.enable = false; auto node = add_ipc_enabled_node (system, config); nano::keypair key1; system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); @@ -3287,7 +3287,7 @@ TEST (rpc, wallet_receivable) { nano::test::system system; nano::node_config config; - config.backlog_population.enable = false; + config.backlog_scan.enable = false; auto node = add_ipc_enabled_node (system, config); nano::keypair key1; system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); @@ -4351,7 +4351,7 @@ TEST (rpc, populate_backlog) nano::test::system system; nano::node_config node_config = system.default_config (); // Disable automatic backlog population - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto node = add_ipc_enabled_node (system, node_config); // Create and process a block that won't get automatically scheduled for confirmation diff --git a/nano/slow_test/node.cpp b/nano/slow_test/node.cpp index 2736d7dd06..75bf0ea2ff 100644 --- a/nano/slow_test/node.cpp +++ b/nano/slow_test/node.cpp @@ -643,7 +643,7 @@ TEST (confirmation_height, many_accounts_single_confirmation) nano::test::system system; nano::node_config node_config = system.default_config (); node_config.online_weight_minimum = 100; - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto node = system.add_node (node_config); system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); @@ -726,7 +726,7 @@ TEST (confirmation_height, many_accounts_many_confirmations) nano::test::system system; nano::node_config node_config = system.default_config (); node_config.online_weight_minimum = 100; - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto node = system.add_node (node_config); system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); @@ -799,7 +799,7 @@ TEST (confirmation_height, long_chains) { nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto node = system.add_node (node_config); nano::keypair key1; system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); @@ -944,7 +944,7 @@ TEST (confirmation_height, dynamic_algorithm) { nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto node = system.add_node (node_config); nano::keypair key; system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); @@ -992,7 +992,7 @@ TEST (confirmation_height, many_accounts_send_receive_self) nano::test::system system; nano::node_config node_config = system.default_config (); node_config.online_weight_minimum = 100; - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; node_config.active_elections.size = 400000; nano::node_flags node_flags; auto node = system.add_node (node_config); @@ -1411,7 +1411,7 @@ TEST (telemetry, under_load) { nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; nano::node_flags node_flags; auto node = system.add_node (node_config, node_flags); node_config.peering_port = system.get_available_port (); @@ -1772,7 +1772,7 @@ TEST (node, mass_block_new) { nano::test::system system; nano::node_config node_config = system.default_config (); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto & node = *system.add_node (node_config); node.network_params.network.aec_loop_interval_ms = 500; @@ -2035,7 +2035,7 @@ TEST (node, wallet_create_block_confirm_conflicts) nano::test::system system; nano::block_builder builder; nano::node_config node_config (system.get_available_port ()); - node_config.backlog_population.enable = false; + node_config.backlog_scan.enable = false; auto node = system.add_node (node_config); auto const num_blocks = 10000; @@ -2106,7 +2106,7 @@ TEST (system, block_sequence) config.peering_port = system.get_available_port (); // config.bandwidth_limit = 16 * 1024; config.enable_voting = true; - config.backlog_population.enable = false; + config.backlog_scan.enable = false; nano::node_flags flags; flags.disable_max_peers_per_ip = true; flags.disable_ongoing_bootstrap = true; diff --git a/nano/slow_test/vote_cache.cpp b/nano/slow_test/vote_cache.cpp index f7d8e61bb9..9f70affa9a 100644 --- a/nano/slow_test/vote_cache.cpp +++ b/nano/slow_test/vote_cache.cpp @@ -134,7 +134,7 @@ TEST (vote_cache, perf_singlethreaded) nano::test::system system; nano::node_flags flags; nano::node_config config = system.default_config (); - config.backlog_population.enable = false; + config.backlog_scan.enable = false; auto & node = *system.add_node (config, flags); const int rep_count = 50; @@ -193,7 +193,7 @@ TEST (vote_cache, perf_multithreaded) nano::test::system system; nano::node_flags flags; nano::node_config config = system.default_config (); - config.backlog_population.enable = false; + config.backlog_scan.enable = false; auto & node = *system.add_node (config, flags); const int thread_count = 12; From 9d131be9548d1e790fa5c5738bdfc1208a9d38d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Fri, 11 Oct 2024 16:24:50 +0200 Subject: [PATCH 4/9] Backlog scan callback event --- nano/core_test/backlog.cpp | 5 ++--- nano/node/backlog_scan.cpp | 9 +++------ nano/node/backlog_scan.hpp | 16 +++++++++++----- nano/node/node.cpp | 7 ++++++- 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/nano/core_test/backlog.cpp b/nano/core_test/backlog.cpp index 9de81919d0..ab6bd0fdff 100644 --- a/nano/core_test/backlog.cpp +++ b/nano/core_test/backlog.cpp @@ -22,10 +22,9 @@ TEST (backlog, population) nano::test::system system{}; auto & node = *system.add_node (); - node.backlog_scan.activate_callback.add ([&] (nano::secure::transaction const & transaction, nano::account const & account) { + node.backlog_scan.activated.add ([&] (nano::secure::transaction const & transaction, auto const & info) { nano::lock_guard lock{ mutex }; - - activated.insert (account); + activated.insert (info.account); }); auto blocks = nano::test::setup_independent_blocks (system, node, 256); diff --git a/nano/node/backlog_scan.cpp b/nano/node/backlog_scan.cpp index 8ba28a0659..b6951f4c7d 100644 --- a/nano/node/backlog_scan.cpp +++ b/nano/node/backlog_scan.cpp @@ -8,9 +8,8 @@ #include #include -nano::backlog_scan::backlog_scan (backlog_scan_config const & config_a, nano::scheduler::component & schedulers, nano::ledger & ledger, nano::stats & stats_a) : +nano::backlog_scan::backlog_scan (backlog_scan_config const & config_a, nano::ledger & ledger, nano::stats & stats_a) : config{ config_a }, - schedulers{ schedulers }, ledger{ ledger }, stats{ stats_a } { @@ -135,10 +134,8 @@ void nano::backlog_scan::activate (secure::transaction const & transaction, nano { stats.inc (nano::stat::type::backlog, nano::stat::detail::activated); - activate_callback.notify (transaction, account); - - schedulers.optimistic.activate (account, account_info, conf_info); - schedulers.priority.activate (transaction, account, account_info, conf_info); + activated_info info{ account, account_info, conf_info }; + activated.notify (transaction, info); } } diff --git a/nano/node/backlog_scan.hpp b/nano/node/backlog_scan.hpp index f9b2ba8aad..756c6deb27 100644 --- a/nano/node/backlog_scan.hpp +++ b/nano/node/backlog_scan.hpp @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include @@ -30,7 +30,7 @@ class backlog_scan_config final class backlog_scan final { public: - backlog_scan (backlog_scan_config const &, nano::scheduler::component &, nano::ledger &, nano::stats &); + backlog_scan (backlog_scan_config const &, nano::ledger &, nano::stats &); ~backlog_scan (); void start (); @@ -43,15 +43,21 @@ class backlog_scan final void notify (); public: + struct activated_info + { + nano::account account; + nano::account_info account_info; + nano::confirmation_height_info conf_info; + }; + /** * Callback called for each backlogged account */ - using callback_t = nano::observer_set; - callback_t activate_callback; + using callback_t = nano::observer_set; + callback_t activated; private: // Dependencies backlog_scan_config const & config; - nano::scheduler::component & schedulers; nano::ledger & ledger; nano::stats & stats; diff --git a/nano/node/node.cpp b/nano/node/node.cpp index 6fd309447b..4ef0323929 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -154,7 +154,7 @@ nano::node::node (std::shared_ptr io_ctx_a, std::filesy aggregator_impl{ std::make_unique (config.request_aggregator, *this, stats, generator, final_generator, history, ledger, wallets, vote_router) }, aggregator{ *aggregator_impl }, wallets (wallets_store.init_error (), *this), - backlog_scan_impl{ std::make_unique (config.backlog_scan, scheduler, ledger, stats) }, + backlog_scan_impl{ std::make_unique (config.backlog_scan, ledger, stats) }, backlog_scan{ *backlog_scan_impl }, bootstrap_server_impl{ std::make_unique (config.bootstrap_server, store, ledger, network_params.network, stats) }, bootstrap_server{ *bootstrap_server_impl }, @@ -180,6 +180,11 @@ nano::node::node (std::shared_ptr io_ctx_a, std::filesy return ledger.weight (rep); }; + backlog_scan.activated.add ([this] (nano::secure::transaction const & transaction, auto const & info) { + scheduler.optimistic.activate (info.account, info.account_info, info.conf_info); + scheduler.priority.activate (transaction, info.account, info.account_info, info.conf_info); + }); + // Republish vote if it is new and the node does not host a principal representative (or close to) vote_router.vote_processed.add ([this] (std::shared_ptr const & vote, nano::vote_source source, std::unordered_map const & results) { bool processed = std::any_of (results.begin (), results.end (), [] (auto const & result) { From 8f6860b6dc853c841b0a973fd6362a81b1c7daa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Fri, 11 Oct 2024 18:46:34 +0200 Subject: [PATCH 5/9] Rename stat to `backlog_scan` --- nano/lib/stats_enums.hpp | 1 + nano/node/backlog_scan.cpp | 7 +++---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nano/lib/stats_enums.hpp b/nano/lib/stats_enums.hpp index b9ce858a17..4feccc2a30 100644 --- a/nano/lib/stats_enums.hpp +++ b/nano/lib/stats_enums.hpp @@ -84,6 +84,7 @@ enum class type active_elections_timeout, active_elections_cancelled, active_elections_cemented, + backlog_scan, backlog, unchecked, election_scheduler, diff --git a/nano/node/backlog_scan.cpp b/nano/node/backlog_scan.cpp index b6951f4c7d..dec7dcab6b 100644 --- a/nano/node/backlog_scan.cpp +++ b/nano/node/backlog_scan.cpp @@ -67,8 +67,7 @@ void nano::backlog_scan::run () { if (predicate ()) { - stats.inc (nano::stat::type::backlog, nano::stat::detail::loop); - + stats.inc (nano::stat::type::backlog_scan, nano::stat::detail::loop); triggered = false; populate_backlog (lock); } @@ -104,7 +103,7 @@ void nano::backlog_scan::populate_backlog (nano::unique_lock & lock for (size_t count = 0; it != end && count < chunk_size && !should_refresh (); ++it, ++count, ++total) { - stats.inc (nano::stat::type::backlog, nano::stat::detail::total); + stats.inc (nano::stat::type::backlog_scan, nano::stat::detail::total); auto const & account = it->first; auto const & account_info = it->second; @@ -132,7 +131,7 @@ void nano::backlog_scan::activate (secure::transaction const & transaction, nano // If conf info is empty then it means then it means nothing is confirmed yet if (conf_info.height < account_info.block_count) { - stats.inc (nano::stat::type::backlog, nano::stat::detail::activated); + stats.inc (nano::stat::type::backlog_scan, nano::stat::detail::activated); activated_info info{ account, account_info, conf_info }; activated.notify (transaction, info); From 7ae7c8a4201ff0cca6399413eedc31b21bb96771 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Sun, 13 Oct 2024 16:56:29 +0200 Subject: [PATCH 6/9] Backlog scan scanned event --- nano/lib/stats_enums.hpp | 1 + nano/node/backlog_scan.cpp | 7 +++++-- nano/node/backlog_scan.hpp | 1 + 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/nano/lib/stats_enums.hpp b/nano/lib/stats_enums.hpp index 4feccc2a30..8281754014 100644 --- a/nano/lib/stats_enums.hpp +++ b/nano/lib/stats_enums.hpp @@ -419,6 +419,7 @@ enum class detail activate_failed, activate_skip, activate_full, + scanned, // active insert, diff --git a/nano/node/backlog_scan.cpp b/nano/node/backlog_scan.cpp index dec7dcab6b..89893dda2c 100644 --- a/nano/node/backlog_scan.cpp +++ b/nano/node/backlog_scan.cpp @@ -128,12 +128,15 @@ void nano::backlog_scan::activate (secure::transaction const & transaction, nano auto const maybe_conf_info = ledger.store.confirmation_height.get (transaction, account); auto const conf_info = maybe_conf_info.value_or (nano::confirmation_height_info{}); + activated_info info{ account, account_info, conf_info }; + + stats.inc (nano::stat::type::backlog_scan, nano::stat::detail::scanned); + scanned.notify (transaction, info); + // If conf info is empty then it means then it means nothing is confirmed yet if (conf_info.height < account_info.block_count) { stats.inc (nano::stat::type::backlog_scan, nano::stat::detail::activated); - - activated_info info{ account, account_info, conf_info }; activated.notify (transaction, info); } } diff --git a/nano/node/backlog_scan.hpp b/nano/node/backlog_scan.hpp index 756c6deb27..2771ab7450 100644 --- a/nano/node/backlog_scan.hpp +++ b/nano/node/backlog_scan.hpp @@ -55,6 +55,7 @@ class backlog_scan final */ using callback_t = nano::observer_set; callback_t activated; + callback_t scanned; private: // Dependencies backlog_scan_config const & config; From bc77ff9a8d0b43897e57d620e9aa2968edb1b154 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Sat, 26 Oct 2024 10:43:45 +0200 Subject: [PATCH 7/9] Use batch event for background backlog scan --- nano/core_test/backlog.cpp | 7 ++- nano/node/backlog_scan.cpp | 94 ++++++++++++++++++-------------- nano/node/backlog_scan.hpp | 24 ++++---- nano/node/node.cpp | 12 +++- nano/node/scheduler/priority.cpp | 9 ++- 5 files changed, 85 insertions(+), 61 deletions(-) diff --git a/nano/core_test/backlog.cpp b/nano/core_test/backlog.cpp index ab6bd0fdff..7e1cf5da1c 100644 --- a/nano/core_test/backlog.cpp +++ b/nano/core_test/backlog.cpp @@ -22,9 +22,12 @@ TEST (backlog, population) nano::test::system system{}; auto & node = *system.add_node (); - node.backlog_scan.activated.add ([&] (nano::secure::transaction const & transaction, auto const & info) { + node.backlog_scan.batch_activated.add ([&] (auto const & batch) { nano::lock_guard lock{ mutex }; - activated.insert (info.account); + for (auto const & info : batch) + { + activated.insert (info.account); + } }); auto blocks = nano::test::setup_independent_blocks (system, node, 256); diff --git a/nano/node/backlog_scan.cpp b/nano/node/backlog_scan.cpp index 89893dda2c..9d922c042d 100644 --- a/nano/node/backlog_scan.cpp +++ b/nano/node/backlog_scan.cpp @@ -8,10 +8,11 @@ #include #include -nano::backlog_scan::backlog_scan (backlog_scan_config const & config_a, nano::ledger & ledger, nano::stats & stats_a) : +nano::backlog_scan::backlog_scan (backlog_scan_config const & config_a, nano::ledger & ledger_a, nano::stats & stats_a) : config{ config_a }, - ledger{ ledger }, - stats{ stats_a } + ledger{ ledger_a }, + stats{ stats_a }, + limiter{ config.batch_size * config.frequency } { } @@ -69,76 +70,85 @@ void nano::backlog_scan::run () { stats.inc (nano::stat::type::backlog_scan, nano::stat::detail::loop); triggered = false; - populate_backlog (lock); + populate_backlog (lock); // Does a single iteration over all accounts + debug_assert (lock.owns_lock ()); + } + else + { + condition.wait (lock, [this] () { + return stopped || predicate (); + }); } - - condition.wait (lock, [this] () { - return stopped || predicate (); - }); } } void nano::backlog_scan::populate_backlog (nano::unique_lock & lock) { - debug_assert (config.frequency > 0); + uint64_t total = 0; - const auto chunk_size = config.batch_size / config.frequency; - auto done = false; nano::account next = 0; - uint64_t total = 0; + bool done = false; while (!stopped && !done) { + // Wait for the rate limiter + while (!limiter.should_pass (config.batch_size)) + { + condition.wait_for (lock, std::chrono::milliseconds{ 1000 / config.frequency / 2 }); + if (stopped) + { + return; + } + } + lock.unlock (); + std::deque scanned; + std::deque activated; { auto transaction = ledger.tx_begin_read (); auto it = ledger.store.account.begin (transaction, next); auto const end = ledger.store.account.end (transaction); - auto should_refresh = [&transaction] () { - auto cutoff = std::chrono::steady_clock::now () - 100ms; // TODO: Make this configurable - return transaction.timestamp () < cutoff; - }; - - for (size_t count = 0; it != end && count < chunk_size && !should_refresh (); ++it, ++count, ++total) + for (size_t count = 0; it != end && count < config.batch_size; ++it, ++count, ++total) { stats.inc (nano::stat::type::backlog_scan, nano::stat::detail::total); - auto const & account = it->first; - auto const & account_info = it->second; + auto const [account, account_info] = *it; + auto const maybe_conf_info = ledger.store.confirmation_height.get (transaction, account); + auto const conf_info = maybe_conf_info.value_or (nano::confirmation_height_info{}); + + activated_info info{ account, account_info, conf_info }; - activate (transaction, account, account_info); + scanned.push_back (info); + if (conf_info.height < account_info.block_count) + { + activated.push_back (info); + } - next = account.number () + 1; + next = account.number () + 1; // TODO: Prevent account overflow } - done = ledger.store.account.begin (transaction, next) == end; + done = (it == end); } - lock.lock (); + stats.add (nano::stat::type::backlog_scan, nano::stat::detail::scanned, scanned.size ()); + stats.add (nano::stat::type::backlog_scan, nano::stat::detail::activated, activated.size ()); - // Give the rest of the node time to progress without holding database lock - condition.wait_for (lock, std::chrono::milliseconds{ 1000 / config.frequency }); + // Notify about scanned and activated accounts without holding database transaction + batch_scanned.notify (scanned); + batch_activated.notify (activated); + + lock.lock (); } } -void nano::backlog_scan::activate (secure::transaction const & transaction, nano::account const & account, nano::account_info const & account_info) +nano::container_info nano::backlog_scan::container_info () const { - auto const maybe_conf_info = ledger.store.confirmation_height.get (transaction, account); - auto const conf_info = maybe_conf_info.value_or (nano::confirmation_height_info{}); - - activated_info info{ account, account_info, conf_info }; - - stats.inc (nano::stat::type::backlog_scan, nano::stat::detail::scanned); - scanned.notify (transaction, info); - - // If conf info is empty then it means then it means nothing is confirmed yet - if (conf_info.height < account_info.block_count) - { - stats.inc (nano::stat::type::backlog_scan, nano::stat::detail::activated); - activated.notify (transaction, info); - } + nano::lock_guard guard{ mutex }; + nano::container_info info; + info.put ("limiter", limiter.size ()); + return info; } /* @@ -149,7 +159,7 @@ nano::error nano::backlog_scan_config::serialize (nano::tomlconfig & toml) const { toml.put ("enable", enable, "Control if ongoing backlog population is enabled. If not, backlog population can still be triggered by RPC \ntype:bool"); toml.put ("batch_size", batch_size, "Number of accounts per second to process when doing backlog population scan. Increasing this value will help unconfirmed frontiers get into election prioritization queue faster, however it will also increase resource usage. \ntype:uint"); - toml.put ("frequency", frequency, "Backlog scan divides the scan into smaller batches, number of which is controlled by this value. Higher frequency helps to utilize resources more uniformly, however it also introduces more overhead. The resulting number of accounts per single batch is `backlog_scan_batch_size / backlog_scan_frequency` \ntype:uint"); + toml.put ("frequency", frequency, "Number of batches to process per second. Higher frequency and smaller batch size helps to utilize resources more uniformly, however it also introduces more overhead. Use 0 to process as fast as possible, but be aware that it may consume a lot of resources. \ntype:uint"); return toml.get_error (); } diff --git a/nano/node/backlog_scan.hpp b/nano/node/backlog_scan.hpp index 2771ab7450..9f65231500 100644 --- a/nano/node/backlog_scan.hpp +++ b/nano/node/backlog_scan.hpp @@ -3,11 +3,13 @@ #include #include #include +#include #include #include #include #include +#include #include namespace nano @@ -21,10 +23,10 @@ class backlog_scan_config final public: /** Control if ongoing backlog population is enabled. If not, backlog population can still be triggered by RPC */ bool enable{ true }; - /** Number of accounts per second to process. Number of accounts per single batch is this value divided by `frequency` */ - unsigned batch_size{ 10 * 1000 }; - /** Number of batches to run per second. Batches run in 1 second / `frequency` intervals */ - unsigned frequency{ 10 }; + /** Number of accounts per second to process. */ + size_t batch_size{ 1000 }; + /** Number of batches to run per second. */ + size_t frequency{ 10 }; }; class backlog_scan final @@ -42,6 +44,8 @@ class backlog_scan final /** Notify about AEC vacancy */ void notify (); + nano::container_info container_info () const; + public: struct activated_info { @@ -50,12 +54,9 @@ class backlog_scan final nano::confirmation_height_info conf_info; }; - /** - * Callback called for each backlogged account - */ - using callback_t = nano::observer_set; - callback_t activated; - callback_t scanned; + using batch_event_t = nano::observer_set>; + batch_event_t batch_scanned; // Accounts scanned but not activated + batch_event_t batch_activated; // Accounts activated private: // Dependencies backlog_scan_config const & config; @@ -66,9 +67,10 @@ class backlog_scan final void run (); bool predicate () const; void populate_backlog (nano::unique_lock & lock); - void activate (secure::transaction const &, nano::account const &, nano::account_info const &); private: + nano::rate_limiter limiter; + /** This is a manual trigger, the ongoing backlog population does not use this. * It can be triggered even when backlog population (frontiers confirmation) is disabled. */ bool triggered{ false }; diff --git a/nano/node/node.cpp b/nano/node/node.cpp index 4ef0323929..32ddf699bd 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -180,9 +180,14 @@ nano::node::node (std::shared_ptr io_ctx_a, std::filesy return ledger.weight (rep); }; - backlog_scan.activated.add ([this] (nano::secure::transaction const & transaction, auto const & info) { - scheduler.optimistic.activate (info.account, info.account_info, info.conf_info); - scheduler.priority.activate (transaction, info.account, info.account_info, info.conf_info); + // TODO: Hook this direclty in the schedulers + backlog_scan.batch_activated.add ([this] (auto const & batch) { + auto transaction = ledger.tx_begin_read (); + for (auto const & info : batch) + { + scheduler.optimistic.activate (info.account, info.account_info, info.conf_info); + scheduler.priority.activate (transaction, info.account, info.account_info, info.conf_info); + } }); // Republish vote if it is new and the node does not host a principal representative (or close to) @@ -1198,6 +1203,7 @@ nano::container_info nano::node::container_info () const info.add ("rep_tiers", rep_tiers.container_info ()); info.add ("message_processor", message_processor.container_info ()); info.add ("bandwidth", outbound_limiter.container_info ()); + info.add ("backlog_scan", backlog_scan.container_info ()); return info; } diff --git a/nano/node/scheduler/priority.cpp b/nano/node/scheduler/priority.cpp index 6bc9091e64..4f63b43b9e 100644 --- a/nano/node/scheduler/priority.cpp +++ b/nano/node/scheduler/priority.cpp @@ -135,9 +135,12 @@ bool nano::scheduler::priority::activate (secure::transaction const & transactio { debug_assert (conf_info.frontier != account_info.head); - auto hash = conf_info.height == 0 ? account_info.open_block : ledger.any.block_successor (transaction, conf_info.frontier).value (); - auto block = ledger.any.block_get (transaction, hash); - release_assert (block != nullptr); + auto const hash = conf_info.height == 0 ? account_info.open_block : ledger.any.block_successor (transaction, conf_info.frontier).value_or (0); + auto const block = ledger.any.block_get (transaction, hash); + if (!block) + { + return false; // Not activated + } if (ledger.dependents_confirmed (transaction, *block)) { From afd5949afd24b25eada28708e3b8592acc407c13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Sat, 26 Oct 2024 11:12:48 +0200 Subject: [PATCH 8/9] Use saturation arithmetic for incrementing account numbers --- nano/core_test/numbers.cpp | 95 +++++++++++++++++++++++++++ nano/lib/numbers.hpp | 27 ++++++++ nano/node/backlog_scan.cpp | 2 +- nano/node/bootstrap/crawlers.hpp | 2 +- nano/node/bootstrap/database_scan.cpp | 4 +- nano/node/json_handler.cpp | 4 +- nano/node/node.cpp | 2 +- 7 files changed, 129 insertions(+), 7 deletions(-) diff --git a/nano/core_test/numbers.cpp b/nano/core_test/numbers.cpp index ec8bed3f4e..7b6e77015e 100644 --- a/nano/core_test/numbers.cpp +++ b/nano/core_test/numbers.cpp @@ -649,3 +649,98 @@ TEST (uint512_union, hash) } } } + +TEST (sat_math, add_sat) +{ + // Test uint128_t + { + nano::uint128_t max = std::numeric_limits::max (); + nano::uint128_t one = 1; + nano::uint128_t large_val = max - 100; + + // Normal addition + ASSERT_EQ (nano::add_sat (one, one), nano::uint128_t (2)); + + // Saturation at max + ASSERT_EQ (nano::add_sat (max, one), max); + ASSERT_EQ (nano::add_sat (large_val, nano::uint128_t (200)), max); + ASSERT_EQ (nano::add_sat (max, max), max); + } + // Test uint256_t + { + nano::uint256_t max = std::numeric_limits::max (); + nano::uint256_t one = 1; + nano::uint256_t large_val = max - 100; + + // Normal addition + ASSERT_EQ (nano::add_sat (one, one), nano::uint256_t (2)); + + // Saturation at max + ASSERT_EQ (nano::add_sat (max, one), max); + ASSERT_EQ (nano::add_sat (large_val, nano::uint256_t (200)), max); + ASSERT_EQ (nano::add_sat (max, max), max); + } + // Test uint512_t + { + nano::uint512_t max = std::numeric_limits::max (); + nano::uint512_t one = 1; + nano::uint512_t large_val = max - 100; + + // Normal addition + ASSERT_EQ (nano::add_sat (one, one), nano::uint512_t (2)); + + // Saturation at max + ASSERT_EQ (nano::add_sat (max, one), max); + ASSERT_EQ (nano::add_sat (large_val, nano::uint512_t (200)), max); + ASSERT_EQ (nano::add_sat (max, max), max); + } +} + +TEST (sat_math, sub_sat) +{ + // Test uint128_t + { + nano::uint128_t max = std::numeric_limits::max (); + nano::uint128_t min = std::numeric_limits::min (); + nano::uint128_t one = 1; + nano::uint128_t hundred (100); + + // Normal subtraction + ASSERT_EQ (nano::sub_sat (hundred, one), nano::uint128_t (99)); + + // Saturation at min + ASSERT_EQ (nano::sub_sat (min, one), min); + ASSERT_EQ (nano::sub_sat (hundred, nano::uint128_t (200)), min); + ASSERT_EQ (nano::sub_sat (min, max), min); + } + // Test uint256_t + { + nano::uint256_t max = std::numeric_limits::max (); + nano::uint256_t min = std::numeric_limits::min (); + nano::uint256_t one = 1; + nano::uint256_t hundred (100); + + // Normal subtraction + ASSERT_EQ (nano::sub_sat (hundred, one), nano::uint256_t (99)); + + // Saturation at min + ASSERT_EQ (nano::sub_sat (min, one), min); + ASSERT_EQ (nano::sub_sat (hundred, nano::uint256_t (200)), min); + ASSERT_EQ (nano::sub_sat (min, max), min); + } + // Test uint512_t + { + nano::uint512_t max = std::numeric_limits::max (); + nano::uint512_t min = std::numeric_limits::min (); + nano::uint512_t one = 1; + nano::uint512_t hundred (100); + + // Normal subtraction + ASSERT_EQ (nano::sub_sat (hundred, one), nano::uint512_t (99)); + + // Saturation at min + ASSERT_EQ (nano::sub_sat (min, one), min); + ASSERT_EQ (nano::sub_sat (hundred, nano::uint512_t (200)), min); + ASSERT_EQ (nano::sub_sat (min, max), min); + } +} \ No newline at end of file diff --git a/nano/lib/numbers.hpp b/nano/lib/numbers.hpp index ccf42ba357..b32c32075b 100644 --- a/nano/lib/numbers.hpp +++ b/nano/lib/numbers.hpp @@ -507,6 +507,33 @@ namespace difficulty uint64_t from_multiplier (double const, uint64_t const); double to_multiplier (uint64_t const, uint64_t const); } + +/** + * Add to or substract from a value without overflow + * TODO: C++26 replace with std::add_sat and std::sub_sat + */ +template +T add_sat (T const & value, T const & diff) noexcept +{ + static_assert (std::numeric_limits::is_specialized, "std::numeric_limits must be specialized"); + return (value > std::numeric_limits::max () - diff) ? std::numeric_limits::max () : value + diff; +} +template +T sub_sat (T const & value, T const & diff) noexcept +{ + static_assert (std::numeric_limits::is_specialized, "std::numeric_limits must be specialized"); + return (value < std::numeric_limits::min () + diff) ? std::numeric_limits::min () : value - diff; +} +template +T inc_sat (T const & value) noexcept +{ + return add_sat (value, static_cast (1)); +} +template +T dec_sat (T const & value) noexcept +{ + return sub_sat (value, static_cast (1)); +} } /* diff --git a/nano/node/backlog_scan.cpp b/nano/node/backlog_scan.cpp index 9d922c042d..880ae8d052 100644 --- a/nano/node/backlog_scan.cpp +++ b/nano/node/backlog_scan.cpp @@ -126,7 +126,7 @@ void nano::backlog_scan::populate_backlog (nano::unique_lock & lock activated.push_back (info); } - next = account.number () + 1; // TODO: Prevent account overflow + next = inc_sat (account.number ()); } done = (it == end); diff --git a/nano/node/bootstrap/crawlers.hpp b/nano/node/bootstrap/crawlers.hpp index a29d53fa9a..dec7ac8de1 100644 --- a/nano/node/bootstrap/crawlers.hpp +++ b/nano/node/bootstrap/crawlers.hpp @@ -134,7 +134,7 @@ struct pending_database_crawler if (it != end) { // If that fails, perform a fresh lookup - seek (starting_account.number () + 1); + seek (inc_sat (starting_account.number ())); } update_current (); diff --git a/nano/node/bootstrap/database_scan.cpp b/nano/node/bootstrap/database_scan.cpp index dc88a9b308..0135129392 100644 --- a/nano/node/bootstrap/database_scan.cpp +++ b/nano/node/bootstrap/database_scan.cpp @@ -78,7 +78,7 @@ std::deque nano::bootstrap::account_database_scanner::next_batch { auto const & [account, info] = crawler.current.value (); result.push_back (account); - next = account.number () + 1; // TODO: Handle account number overflow + next = inc_sat (account.number ()); } // Empty current value indicates the end of the table @@ -106,7 +106,7 @@ std::deque nano::bootstrap::pending_database_scanner::next_batch { auto const & [key, info] = crawler.current.value (); result.push_back (key.account); - next = key.account.number () + 1; // TODO: Handle account number overflow + next = inc_sat (key.account.number ()); } // Empty current value indicates the end of the table diff --git a/nano/node/json_handler.cpp b/nano/node/json_handler.cpp index af496a963e..681fcaba8a 100644 --- a/nano/node/json_handler.cpp +++ b/nano/node/json_handler.cpp @@ -2173,7 +2173,7 @@ void nano::json_handler::delegators () { auto transaction (node.ledger.tx_begin_read ()); boost::property_tree::ptree delegators; - for (auto i (node.store.account.begin (transaction, start_account.number () + 1)), n (node.store.account.end (transaction)); i != n && delegators.size () < count; ++i) + for (auto i (node.store.account.begin (transaction, inc_sat (start_account.number ()))), n (node.store.account.end (transaction)); i != n && delegators.size () < count; ++i) { nano::account_info const & info (i->second); if (info.representative == representative) @@ -4189,7 +4189,7 @@ void nano::json_handler::unopened () break; } // Skip existing accounts - iterator = node.store.pending.begin (transaction, nano::pending_key (account.number () + 1, 0)); + iterator = node.store.pending.begin (transaction, nano::pending_key (inc_sat (account.number ()), 0)); } else { diff --git a/nano/node/node.cpp b/nano/node/node.cpp index 32ddf699bd..62ee9680d7 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -856,7 +856,7 @@ bool nano::node::collect_ledger_pruning_targets (std::deque & read_operations += depth; if (read_operations >= batch_read_size_a) { - last_account_a = account.number () + 1; + last_account_a = inc_sat (account.number ()); finish_transaction = true; } else From 59b053287db64ddda537ca2aef887536e4371f91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Tue, 26 Nov 2024 19:37:03 +0100 Subject: [PATCH 9/9] Directly configurable rate limit --- nano/core_test/toml.cpp | 6 +++--- nano/node/backlog_scan.cpp | 11 ++++++----- nano/node/backlog_scan.hpp | 4 ++-- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/nano/core_test/toml.cpp b/nano/core_test/toml.cpp index 1cbca2edc5..9d6e2124ab 100644 --- a/nano/core_test/toml.cpp +++ b/nano/core_test/toml.cpp @@ -199,7 +199,7 @@ TEST (toml, daemon_config_deserialize_defaults) ASSERT_EQ (conf.node.max_unchecked_blocks, defaults.node.max_unchecked_blocks); ASSERT_EQ (conf.node.backlog_scan.enable, defaults.node.backlog_scan.enable); ASSERT_EQ (conf.node.backlog_scan.batch_size, defaults.node.backlog_scan.batch_size); - ASSERT_EQ (conf.node.backlog_scan.frequency, defaults.node.backlog_scan.frequency); + ASSERT_EQ (conf.node.backlog_scan.rate_limit, defaults.node.backlog_scan.rate_limit); ASSERT_EQ (conf.node.enable_upnp, defaults.node.enable_upnp); ASSERT_EQ (conf.node.websocket_config.enabled, defaults.node.websocket_config.enabled); @@ -468,7 +468,7 @@ TEST (toml, daemon_config_deserialize_no_defaults) [node.backlog_scan] enable = false batch_size = 999 - frequency = 999 + rate_limit = 999 [node.block_processor] max_peer_queue = 999 @@ -706,7 +706,7 @@ TEST (toml, daemon_config_deserialize_no_defaults) ASSERT_NE (conf.node.request_aggregator_threads, defaults.node.request_aggregator_threads); ASSERT_NE (conf.node.backlog_scan.enable, defaults.node.backlog_scan.enable); ASSERT_NE (conf.node.backlog_scan.batch_size, defaults.node.backlog_scan.batch_size); - ASSERT_NE (conf.node.backlog_scan.frequency, defaults.node.backlog_scan.frequency); + ASSERT_NE (conf.node.backlog_scan.rate_limit, defaults.node.backlog_scan.rate_limit); ASSERT_NE (conf.node.enable_upnp, defaults.node.enable_upnp); ASSERT_NE (conf.node.websocket_config.enabled, defaults.node.websocket_config.enabled); diff --git a/nano/node/backlog_scan.cpp b/nano/node/backlog_scan.cpp index 880ae8d052..c725c04b1f 100644 --- a/nano/node/backlog_scan.cpp +++ b/nano/node/backlog_scan.cpp @@ -12,7 +12,7 @@ nano::backlog_scan::backlog_scan (backlog_scan_config const & config_a, nano::le config{ config_a }, ledger{ ledger_a }, stats{ stats_a }, - limiter{ config.batch_size * config.frequency } + limiter{ config.rate_limit } { } @@ -93,7 +93,8 @@ void nano::backlog_scan::populate_backlog (nano::unique_lock & lock // Wait for the rate limiter while (!limiter.should_pass (config.batch_size)) { - condition.wait_for (lock, std::chrono::milliseconds{ 1000 / config.frequency / 2 }); + std::chrono::milliseconds const wait_time{ 1000 / std::max ((config.rate_limit / config.batch_size), size_t{ 1 }) / 2 }; + condition.wait_for (lock, std::max (wait_time, 10ms)); if (stopped) { return; @@ -158,8 +159,8 @@ nano::container_info nano::backlog_scan::container_info () const nano::error nano::backlog_scan_config::serialize (nano::tomlconfig & toml) const { toml.put ("enable", enable, "Control if ongoing backlog population is enabled. If not, backlog population can still be triggered by RPC \ntype:bool"); - toml.put ("batch_size", batch_size, "Number of accounts per second to process when doing backlog population scan. Increasing this value will help unconfirmed frontiers get into election prioritization queue faster, however it will also increase resource usage. \ntype:uint"); - toml.put ("frequency", frequency, "Number of batches to process per second. Higher frequency and smaller batch size helps to utilize resources more uniformly, however it also introduces more overhead. Use 0 to process as fast as possible, but be aware that it may consume a lot of resources. \ntype:uint"); + toml.put ("batch_size", batch_size, "Size of a single batch. Larger batches reduce overhead, but may put more pressure on other node components. \ntype:uint"); + toml.put ("rate_limit", rate_limit, "Number of accounts per second to process when doing backlog population scan. Increasing this value will help unconfirmed frontiers get into election prioritization queue faster. Use 0 to process as fast as possible, but be aware that it may consume a lot of resources. \ntype:uint"); return toml.get_error (); } @@ -168,7 +169,7 @@ nano::error nano::backlog_scan_config::deserialize (nano::tomlconfig & toml) { toml.get ("enable", enable); toml.get ("batch_size", batch_size); - toml.get ("frequency", frequency); + toml.get ("rate_limit", rate_limit); return toml.get_error (); } diff --git a/nano/node/backlog_scan.hpp b/nano/node/backlog_scan.hpp index 9f65231500..5ff41629a1 100644 --- a/nano/node/backlog_scan.hpp +++ b/nano/node/backlog_scan.hpp @@ -23,10 +23,10 @@ class backlog_scan_config final public: /** Control if ongoing backlog population is enabled. If not, backlog population can still be triggered by RPC */ bool enable{ true }; + /** Number of accounts to scan per second. */ + size_t rate_limit{ 10000 }; /** Number of accounts per second to process. */ size_t batch_size{ 1000 }; - /** Number of batches to run per second. */ - size_t frequency{ 10 }; }; class backlog_scan final