diff --git a/nano/core_test/toml.cpp b/nano/core_test/toml.cpp index edb1e9226c..744e70acd8 100644 --- a/nano/core_test/toml.cpp +++ b/nano/core_test/toml.cpp @@ -275,6 +275,7 @@ TEST (toml, daemon_config_deserialize_defaults) ASSERT_EQ (conf.node.hinted_scheduler.hinting_threshold_percent, defaults.node.hinted_scheduler.hinting_threshold_percent); ASSERT_EQ (conf.node.hinted_scheduler.check_interval.count (), defaults.node.hinted_scheduler.check_interval.count ()); ASSERT_EQ (conf.node.hinted_scheduler.block_cooldown.count (), defaults.node.hinted_scheduler.block_cooldown.count ()); + ASSERT_EQ (conf.node.hinted_scheduler.vacancy_threshold_percent, defaults.node.hinted_scheduler.vacancy_threshold_percent); ASSERT_EQ (conf.node.vote_cache.max_size, defaults.node.vote_cache.max_size); ASSERT_EQ (conf.node.vote_cache.max_voters, defaults.node.vote_cache.max_voters); @@ -543,6 +544,7 @@ TEST (toml, daemon_config_deserialize_no_defaults) hinting_threshold = 99 check_interval = 999 block_cooldown = 999 + vacancy_threshold = 99 [node.rocksdb] enable = true @@ -720,6 +722,7 @@ TEST (toml, daemon_config_deserialize_no_defaults) ASSERT_NE (conf.node.hinted_scheduler.hinting_threshold_percent, defaults.node.hinted_scheduler.hinting_threshold_percent); ASSERT_NE (conf.node.hinted_scheduler.check_interval.count (), defaults.node.hinted_scheduler.check_interval.count ()); ASSERT_NE (conf.node.hinted_scheduler.block_cooldown.count (), defaults.node.hinted_scheduler.block_cooldown.count ()); + ASSERT_NE (conf.node.hinted_scheduler.vacancy_threshold_percent, defaults.node.hinted_scheduler.vacancy_threshold_percent); ASSERT_NE (conf.node.vote_cache.max_size, defaults.node.vote_cache.max_size); ASSERT_NE (conf.node.vote_cache.max_voters, defaults.node.vote_cache.max_voters); diff --git a/nano/core_test/vote_cache.cpp b/nano/core_test/vote_cache.cpp index f94dda0679..b4ac518af8 100644 --- a/nano/core_test/vote_cache.cpp +++ b/nano/core_test/vote_cache.cpp @@ -36,8 +36,9 @@ nano::keypair create_rep (nano::uint128_t weight) TEST (vote_cache, construction) { + nano::test::system system; nano::vote_cache_config cfg; - nano::vote_cache vote_cache{ cfg }; + nano::vote_cache vote_cache{ cfg, system.stats }; ASSERT_EQ (0, vote_cache.size ()); ASSERT_TRUE (vote_cache.empty ()); auto hash1 = nano::test::random_hash (); @@ -49,8 +50,9 @@ TEST (vote_cache, construction) */ TEST (vote_cache, insert_one_hash) { + nano::test::system system; nano::vote_cache_config cfg; - nano::vote_cache vote_cache{ cfg }; + nano::vote_cache vote_cache{ cfg, system.stats }; vote_cache.rep_weight_query = rep_weight_query (); auto rep1 = create_rep (7); auto hash1 = nano::test::random_hash (); @@ -79,8 +81,9 @@ TEST (vote_cache, insert_one_hash) */ TEST (vote_cache, insert_one_hash_many_votes) { + nano::test::system system; nano::vote_cache_config cfg; - nano::vote_cache vote_cache{ cfg }; + nano::vote_cache vote_cache{ cfg, system.stats }; vote_cache.rep_weight_query = rep_weight_query (); auto hash1 = nano::test::random_hash (); auto rep1 = create_rep (7); @@ -114,8 +117,9 @@ TEST (vote_cache, insert_one_hash_many_votes) */ TEST (vote_cache, insert_many_hashes_many_votes) { + nano::test::system system; nano::vote_cache_config cfg; - nano::vote_cache vote_cache{ cfg }; + nano::vote_cache vote_cache{ cfg, system.stats }; vote_cache.rep_weight_query = rep_weight_query (); // There will be 3 random hashes to vote for auto hash1 = nano::test::random_hash (); @@ -194,8 +198,9 @@ TEST (vote_cache, insert_many_hashes_many_votes) */ TEST (vote_cache, insert_duplicate) { + nano::test::system system; nano::vote_cache_config cfg; - nano::vote_cache vote_cache{ cfg }; + nano::vote_cache vote_cache{ cfg, system.stats }; vote_cache.rep_weight_query = rep_weight_query (); auto hash1 = nano::test::random_hash (); auto rep1 = create_rep (9); @@ -211,8 +216,9 @@ TEST (vote_cache, insert_duplicate) */ TEST (vote_cache, insert_newer) { + nano::test::system system; nano::vote_cache_config cfg; - nano::vote_cache vote_cache{ cfg }; + nano::vote_cache vote_cache{ cfg, system.stats }; vote_cache.rep_weight_query = rep_weight_query (); auto hash1 = nano::test::random_hash (); auto rep1 = create_rep (9); @@ -236,8 +242,9 @@ TEST (vote_cache, insert_newer) */ TEST (vote_cache, insert_older) { + nano::test::system system; nano::vote_cache_config cfg; - nano::vote_cache vote_cache{ cfg }; + nano::vote_cache vote_cache{ cfg, system.stats }; vote_cache.rep_weight_query = rep_weight_query (); auto hash1 = nano::test::random_hash (); auto rep1 = create_rep (9); @@ -259,8 +266,9 @@ TEST (vote_cache, insert_older) */ TEST (vote_cache, erase) { + nano::test::system system; nano::vote_cache_config cfg; - nano::vote_cache vote_cache{ cfg }; + nano::vote_cache vote_cache{ cfg, system.stats }; vote_cache.rep_weight_query = rep_weight_query (); auto hash1 = nano::test::random_hash (); auto hash2 = nano::test::random_hash (); @@ -298,10 +306,11 @@ TEST (vote_cache, erase) */ TEST (vote_cache, overfill) { + nano::test::system system; // Create a vote cache with max size set to 1024 nano::vote_cache_config cfg; cfg.max_size = 1024; - nano::vote_cache vote_cache{ cfg }; + nano::vote_cache vote_cache{ cfg, system.stats }; vote_cache.rep_weight_query = rep_weight_query (); const int count = 16 * 1024; for (int n = 0; n < count; ++n) @@ -324,8 +333,9 @@ TEST (vote_cache, overfill) */ TEST (vote_cache, overfill_entry) { + nano::test::system system; nano::vote_cache_config cfg; - nano::vote_cache vote_cache{ cfg }; + nano::vote_cache vote_cache{ cfg, system.stats }; vote_cache.rep_weight_query = rep_weight_query (); const int count = 1024; auto hash1 = nano::test::random_hash (); @@ -336,4 +346,40 @@ TEST (vote_cache, overfill_entry) vote_cache.vote (vote1->hashes.front (), vote1); } ASSERT_EQ (1, vote_cache.size ()); +} + +TEST (vote_cache, age_cutoff) +{ + nano::test::system system; + nano::vote_cache_config cfg; + cfg.age_cutoff = std::chrono::seconds{ 3 }; + nano::vote_cache vote_cache{ cfg, system.stats }; + vote_cache.rep_weight_query = rep_weight_query (); + + auto hash1 = nano::test::random_hash (); + auto rep1 = create_rep (9); + auto vote1 = nano::test::make_vote (rep1, { hash1 }, 3); + vote_cache.vote (vote1->hashes.front (), vote1); + ASSERT_EQ (1, vote_cache.size ()); + ASSERT_TRUE (vote_cache.find (hash1)); + + auto tops1 = vote_cache.top (0); + ASSERT_EQ (tops1.size (), 1); + ASSERT_EQ (tops1[0].hash, hash1); + ASSERT_EQ (system.stats.count (nano::stat::type::vote_cache, nano::stat::detail::cleanup), 0); + + // Wait for first cleanup + auto check = [&] () { + // Cleanup is performed periodically when calling `top ()` + vote_cache.top (0); + return system.stats.count (nano::stat::type::vote_cache, nano::stat::detail::cleanup); + }; + ASSERT_TIMELY_EQ (5s, 1, check ()); + + // After first cleanup the entry should still be there + auto tops2 = vote_cache.top (0); + ASSERT_EQ (tops2.size (), 1); + + // After 3 seconds the entry should be removed + ASSERT_TIMELY (5s, vote_cache.top (0).empty ()); } \ No newline at end of file diff --git a/nano/lib/CMakeLists.txt b/nano/lib/CMakeLists.txt index 934cd1452a..6d2043a634 100644 --- a/nano/lib/CMakeLists.txt +++ b/nano/lib/CMakeLists.txt @@ -38,6 +38,7 @@ add_library( errors.hpp errors.cpp id_dispenser.hpp + interval.hpp ipc.hpp ipc.cpp ipc_client.hpp diff --git a/nano/lib/interval.hpp b/nano/lib/interval.hpp new file mode 100644 index 0000000000..263f15ce70 --- /dev/null +++ b/nano/lib/interval.hpp @@ -0,0 +1,30 @@ +#pragma once + +#include + +namespace nano +{ +class interval +{ +public: + explicit interval (std::chrono::milliseconds target) : + target{ target } + { + } + + bool elapsed () + { + auto const now = std::chrono::steady_clock::now (); + if (now - last >= target) + { + last = now; + return true; + } + return false; + } + +private: + std::chrono::milliseconds const target; + std::chrono::steady_clock::time_point last{ std::chrono::steady_clock::now () }; +}; +} \ No newline at end of file diff --git a/nano/lib/stats_enums.hpp b/nano/lib/stats_enums.hpp index 79ef835802..00bc95872f 100644 --- a/nano/lib/stats_enums.hpp +++ b/nano/lib/stats_enums.hpp @@ -63,6 +63,8 @@ enum class detail : uint8_t update, request, broadcast, + cleanup, + top, // processing queue queue, diff --git a/nano/node/node.cpp b/nano/node/node.cpp index aede8116c8..e138f2e2f2 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -182,7 +182,7 @@ nano::node::node (boost::asio::io_context & io_ctx_a, std::filesystem::path cons history{ config.network_params.voting }, vote_uniquer (block_uniquer), confirmation_height_processor (ledger, write_database_queue, config.conf_height_processor_batch_min_time, config.logging, logger, node_initialized_latch, flags.confirmation_height_processor_mode), - vote_cache{ config.vote_cache }, + vote_cache{ config.vote_cache, stats }, generator{ config, ledger, wallets, vote_processor, history, network, stats, /* non-final */ false }, final_generator{ config, ledger, wallets, vote_processor, history, network, stats, /* final */ true }, active (*this, confirmation_height_processor), diff --git a/nano/node/scheduler/hinted.cpp b/nano/node/scheduler/hinted.cpp index 54f5bdf7b8..6bbaff7de9 100644 --- a/nano/node/scheduler/hinted.cpp +++ b/nano/node/scheduler/hinted.cpp @@ -47,7 +47,7 @@ void nano::scheduler::hinted::notify () { // Avoid notifying when there is very little space inside AEC auto const limit = active.limit (nano::election_behavior::hinted); - if (active.vacancy (nano::election_behavior::hinted) >= (limit / 5)) + if (active.vacancy (nano::election_behavior::hinted) >= (limit * config.vacancy_threshold_percent / 100)) { condition.notify_all (); } @@ -107,7 +107,8 @@ void nano::scheduler::hinted::activate (const nano::store::read_transaction & tr else { stats.inc (nano::stat::type::hinting, nano::stat::detail::missing_block); - node.bootstrap_block (current_hash); + + // TODO: Block is missing, bootstrap it } } } @@ -229,6 +230,7 @@ nano::error nano::scheduler::hinted_config::serialize (nano::tomlconfig & toml) toml.put ("hinting_threshold", hinting_threshold_percent, "Percentage of online weight needed to start a hinted election. \ntype:uint32,[0,100]"); toml.put ("check_interval", check_interval.count (), "Interval between scans of the vote cache for possible hinted elections. \ntype:milliseconds"); toml.put ("block_cooldown", block_cooldown.count (), "Cooldown period for blocks that failed to start an election. \ntype:milliseconds"); + toml.put ("vacancy_threshold", vacancy_threshold_percent, "Percentage of available space in the active elections container needed to trigger a scan for hinted elections (before the check interval elapses). \ntype:uint32,[0,100]"); return toml.get_error (); } @@ -245,10 +247,16 @@ nano::error nano::scheduler::hinted_config::deserialize (nano::tomlconfig & toml toml.get ("block_cooldown", block_cooldown_l); block_cooldown = std::chrono::milliseconds{ block_cooldown_l }; + toml.get ("vacancy_threshold", vacancy_threshold_percent); + if (hinting_threshold_percent > 100) { toml.get_error ().set ("hinting_threshold must be a number between 0 and 100"); } + if (vacancy_threshold_percent > 100) + { + toml.get_error ().set ("vacancy_threshold must be a number between 0 and 100"); + } return toml.get_error (); } \ No newline at end of file diff --git a/nano/node/scheduler/hinted.hpp b/nano/node/scheduler/hinted.hpp index 590697c39f..a80bb68c05 100644 --- a/nano/node/scheduler/hinted.hpp +++ b/nano/node/scheduler/hinted.hpp @@ -37,8 +37,9 @@ class hinted_config final public: std::chrono::milliseconds check_interval{ 1000 }; - std::chrono::milliseconds block_cooldown{ 5000 }; + std::chrono::milliseconds block_cooldown{ 10000 }; unsigned hinting_threshold_percent{ 10 }; + unsigned vacancy_threshold_percent{ 20 }; }; /* diff --git a/nano/node/vote_cache.cpp b/nano/node/vote_cache.cpp index 4b3622ad44..b6e2bb8040 100644 --- a/nano/node/vote_cache.cpp +++ b/nano/node/vote_cache.cpp @@ -12,6 +12,16 @@ nano::vote_cache::entry::entry (const nano::block_hash & hash) : } bool nano::vote_cache::entry::vote (const nano::account & representative, const uint64_t & timestamp, const nano::uint128_t & rep_weight, std::size_t max_voters) +{ + bool updated = vote_impl (representative, timestamp, rep_weight, max_voters); + if (updated) + { + last_vote_m = std::chrono::steady_clock::now (); + } + return updated; +} + +bool nano::vote_cache::entry::vote_impl (const nano::account & representative, const uint64_t & timestamp, const nano::uint128_t & rep_weight, std::size_t max_voters) { auto existing = std::find_if (voters_m.begin (), voters_m.end (), [&representative] (auto const & item) { return item.representative == representative; }); if (existing != voters_m.end ()) @@ -92,17 +102,27 @@ std::vector nano::vote_cache::entry::voter return voters_m; } +std::chrono::steady_clock::time_point nano::vote_cache::entry::last_vote () const +{ + return last_vote_m; +} + /* * vote_cache */ -nano::vote_cache::vote_cache (vote_cache_config const & config_a) : - config{ config_a } +nano::vote_cache::vote_cache (vote_cache_config const & config_a, nano::stats & stats_a) : + config{ config_a }, + stats{ stats_a }, + cleanup_interval{ config_a.age_cutoff / 2 } { } void nano::vote_cache::vote (const nano::block_hash & hash, const std::shared_ptr vote) { + // Assert that supplied hash corresponds to a one of the hashes stored in vote + debug_assert (std::find (vote->hashes.begin (), vote->hashes.end (), hash) != vote->hashes.end ()); + auto const representative = vote->account; auto const timestamp = vote->timestamp (); auto const rep_weight = rep_weight_query (representative); @@ -112,12 +132,16 @@ void nano::vote_cache::vote (const nano::block_hash & hash, const std::shared_pt auto & cache_by_hash = cache.get (); if (auto existing = cache_by_hash.find (hash); existing != cache_by_hash.end ()) { + stats.inc (nano::stat::type::vote_cache, nano::stat::detail::update); + cache_by_hash.modify (existing, [this, &representative, ×tamp, &rep_weight] (entry & ent) { ent.vote (representative, timestamp, rep_weight, config.max_voters); }); } else { + stats.inc (nano::stat::type::vote_cache, nano::stat::detail::insert); + entry cache_entry{ hash }; cache_entry.vote (representative, timestamp, rep_weight, config.max_voters); @@ -169,12 +193,19 @@ bool nano::vote_cache::erase (const nano::block_hash & hash) return result; } -std::vector nano::vote_cache::top (const nano::uint128_t & min_tally) const +std::vector nano::vote_cache::top (const nano::uint128_t & min_tally) { + stats.inc (nano::stat::type::vote_cache, nano::stat::detail::top); + std::vector results; { nano::lock_guard lock{ mutex }; + if (cleanup_interval.elapsed ()) + { + cleanup (); + } + for (auto & entry : cache.get ()) { if (entry.tally () < min_tally) @@ -200,7 +231,29 @@ std::vector nano::vote_cache::top (const nano::uint return results; } -std::unique_ptr nano::vote_cache::collect_container_info (const std::string & name) +void nano::vote_cache::cleanup () +{ + debug_assert (!mutex.try_lock ()); + + stats.inc (nano::stat::type::vote_cache, nano::stat::detail::cleanup); + + auto const cutoff = std::chrono::steady_clock::now () - config.age_cutoff; + + auto it = cache.begin (); + while (it != cache.end ()) + { + if (it->last_vote () < cutoff) + { + it = cache.erase (it); + } + else + { + ++it; + } + } +} + +std::unique_ptr nano::vote_cache::collect_container_info (const std::string & name) const { auto composite = std::make_unique (name); composite->add_component (std::make_unique (container_info{ "cache", size (), sizeof (ordered_cache::value_type) })); @@ -215,6 +268,7 @@ nano::error nano::vote_cache_config::serialize (nano::tomlconfig & toml) const { toml.put ("max_size", max_size, "Maximum number of blocks to cache votes for. \ntype:uint64"); toml.put ("max_voters", max_voters, "Maximum number of voters to cache per block. \ntype:uint64"); + toml.put ("age_cutoff", age_cutoff.count (), "Maximum age of votes to keep in cache. \ntype:seconds"); return toml.get_error (); } @@ -224,5 +278,9 @@ nano::error nano::vote_cache_config::deserialize (nano::tomlconfig & toml) toml.get ("max_size", max_size); toml.get ("max_voters", max_voters); + auto age_cutoff_l = age_cutoff.count (); + toml.get ("age_cutoff", age_cutoff_l); + age_cutoff = std::chrono::seconds{ age_cutoff_l }; + return toml.get_error (); } \ No newline at end of file diff --git a/nano/node/vote_cache.hpp b/nano/node/vote_cache.hpp index a076981f7f..4a353446af 100644 --- a/nano/node/vote_cache.hpp +++ b/nano/node/vote_cache.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -38,6 +39,7 @@ class vote_cache_config final public: std::size_t max_size{ 1024 * 128 }; std::size_t max_voters{ 128 }; + std::chrono::seconds age_cutoff{ 5 * 60 }; }; class vote_cache final @@ -63,6 +65,7 @@ class vote_cache final * @return true if current tally changed, false otherwise */ bool vote (nano::account const & representative, uint64_t const & timestamp, nano::uint128_t const & rep_weight, std::size_t max_voters); + /** * Inserts votes stored in this entry into an election */ @@ -73,17 +76,22 @@ class vote_cache final nano::uint128_t tally () const; nano::uint128_t final_tally () const; std::vector voters () const; + std::chrono::steady_clock::time_point last_vote () const; private: + bool vote_impl (nano::account const & representative, uint64_t const & timestamp, nano::uint128_t const & rep_weight, std::size_t max_voters); + nano::block_hash const hash_m; std::vector voters_m; nano::uint128_t tally_m{ 0 }; nano::uint128_t final_tally_m{ 0 }; + + std::chrono::steady_clock::time_point last_vote_m{}; }; public: - explicit vote_cache (vote_cache_config const &); + explicit vote_cache (vote_cache_config const &, nano::stats &); /** * Adds a new vote to cache @@ -115,10 +123,10 @@ class vote_cache final * The blocks are sorted in descending order by final tally, then by tally * @param min_tally minimum tally threshold, entries below with their voting weight below this will be ignored */ - std::vector top (nano::uint128_t const & min_tally) const; + std::vector top (nano::uint128_t const & min_tally); public: // Container info - std::unique_ptr collect_container_info (std::string const & name); + std::unique_ptr collect_container_info (std::string const & name) const; public: /** @@ -126,8 +134,12 @@ class vote_cache final */ std::function rep_weight_query{ [] (nano::account const & rep) { debug_assert (false); return 0; } }; -private: +private: // Dependencies vote_cache_config const & config; + nano::stats & stats; + +private: + void cleanup (); // clang-format off class tag_sequenced {}; @@ -148,5 +160,6 @@ class vote_cache final ordered_cache cache; mutable nano::mutex mutex; + nano::interval cleanup_interval; }; } \ No newline at end of file