diff --git a/nano/core_test/conflicts.cpp b/nano/core_test/conflicts.cpp index b37bd9560b..0d3f3d4210 100644 --- a/nano/core_test/conflicts.cpp +++ b/nano/core_test/conflicts.cpp @@ -179,15 +179,13 @@ TEST (conflicts, add_two) TEST (vote_uniquer, null) { - nano::block_uniquer block_uniquer; - nano::vote_uniquer uniquer (block_uniquer); + nano::vote_uniquer uniquer; ASSERT_EQ (nullptr, uniquer.unique (nullptr)); } TEST (vote_uniquer, vbh_one) { - nano::block_uniquer block_uniquer; - nano::vote_uniquer uniquer (block_uniquer); + nano::vote_uniquer uniquer; nano::keypair key; nano::block_builder builder; auto block = builder @@ -210,8 +208,7 @@ TEST (vote_uniquer, vbh_one) TEST (vote_uniquer, vbh_two) { - nano::block_uniquer block_uniquer; - nano::vote_uniquer uniquer (block_uniquer); + nano::vote_uniquer uniquer; nano::keypair key; nano::block_builder builder; auto block1 = builder @@ -246,8 +243,7 @@ TEST (vote_uniquer, vbh_two) TEST (vote_uniquer, cleanup) { - nano::block_uniquer block_uniquer; - nano::vote_uniquer uniquer (block_uniquer); + nano::vote_uniquer uniquer; nano::keypair key; auto vote1 = std::make_shared (key.pub, key.prv, 0, 0, std::vector{ nano::block_hash{ 0 } }); auto vote2 = std::make_shared (key.pub, key.prv, nano::vote::timestamp_min * 1, 0, std::vector{ nano::block_hash{ 0 } }); @@ -256,10 +252,7 @@ TEST (vote_uniquer, cleanup) vote2.reset (); vote4.reset (); ASSERT_EQ (2, uniquer.size ()); - auto iterations (0); - while (uniquer.size () == 2) - { - auto vote5 (uniquer.unique (vote1)); - ASSERT_LT (iterations++, 200); - } + std::this_thread::sleep_for (nano::block_uniquer::cleanup_cutoff); + auto vote5 = uniquer.unique (vote1); + ASSERT_EQ (1, uniquer.size ()); } diff --git a/nano/core_test/message_deserializer.cpp b/nano/core_test/message_deserializer.cpp index d1e717e466..b2092b5bb8 100644 --- a/nano/core_test/message_deserializer.cpp +++ b/nano/core_test/message_deserializer.cpp @@ -17,7 +17,7 @@ auto message_deserializer_success_checker (message_type & message_original) -> v // Dependencies for the message deserializer. nano::network_filter filter (1); nano::block_uniquer block_uniquer; - nano::vote_uniquer vote_uniquer (block_uniquer); + nano::vote_uniquer vote_uniquer; // Data used to simulate the incoming buffer to be deserialized, the offset tracks how much has been read from the input_source // as the read function is called first to read the header, then called again to read the payload. diff --git a/nano/lib/CMakeLists.txt b/nano/lib/CMakeLists.txt index 6d2043a634..d56f55e8f7 100644 --- a/nano/lib/CMakeLists.txt +++ b/nano/lib/CMakeLists.txt @@ -91,6 +91,7 @@ add_library( tlsconfig.cpp tomlconfig.hpp tomlconfig.cpp + uniquer.hpp utility.hpp utility.cpp walletconfig.hpp diff --git a/nano/lib/blocks.cpp b/nano/lib/blocks.cpp index 8076a400a8..15881b3e50 100644 --- a/nano/lib/blocks.cpp +++ b/nano/lib/blocks.cpp @@ -1864,54 +1864,3 @@ bool nano::block_sideband::deserialize (nano::stream & stream_a, nano::block_typ return result; } - -std::shared_ptr nano::block_uniquer::unique (std::shared_ptr const & block_a) -{ - auto result (block_a); - if (result != nullptr) - { - nano::uint256_union key (block_a->full_hash ()); - nano::lock_guard lock{ mutex }; - auto & existing (blocks[key]); - if (auto block_l = existing.lock ()) - { - result = block_l; - } - else - { - existing = block_a; - } - auto now = std::chrono::steady_clock::now (); - if (cleanup_cutoff < now - cleanup_last) - { - cleanup_last = now; - for (auto i = blocks.begin (), n = blocks.end (); i != n;) - { - if (auto block_l = i->second.lock ()) - { - ++i; - } - else - { - i = blocks.erase (i); - } - } - } - } - return result; -} - -size_t nano::block_uniquer::size () -{ - nano::lock_guard lock{ mutex }; - return blocks.size (); -} - -std::unique_ptr nano::collect_container_info (block_uniquer & block_uniquer, std::string const & name) -{ - auto count = block_uniquer.size (); - auto sizeof_element = sizeof (block_uniquer::value_type); - auto composite = std::make_unique (name); - composite->add_component (std::make_unique (container_info{ "blocks", count, sizeof_element })); - return composite; -} diff --git a/nano/lib/blocks.hpp b/nano/lib/blocks.hpp index 3b779fa63c..5a9ab168bc 100644 --- a/nano/lib/blocks.hpp +++ b/nano/lib/blocks.hpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -399,27 +400,8 @@ class mutable_block_visitor virtual void state_block (nano::state_block &) = 0; virtual ~mutable_block_visitor () = default; }; -/** - * This class serves to find and return unique variants of a block in order to minimize memory usage - */ -class block_uniquer -{ -public: - using value_type = std::pair>; - - std::shared_ptr unique (std::shared_ptr const &); - size_t size (); - -private: - nano::mutex mutex{ mutex_identifier (mutexes::block_uniquer) }; - std::unordered_map, value_type::second_type> blocks; - std::chrono::steady_clock::time_point cleanup_last{ std::chrono::steady_clock::now () }; - -public: - static std::chrono::milliseconds constexpr cleanup_cutoff{ 500 }; -}; -std::unique_ptr collect_container_info (block_uniquer & block_uniquer, std::string const & name); +using block_uniquer = nano::uniquer; std::shared_ptr deserialize_block (nano::stream &); std::shared_ptr deserialize_block (nano::stream &, nano::block_type, nano::block_uniquer * = nullptr); diff --git a/nano/lib/uniquer.hpp b/nano/lib/uniquer.hpp new file mode 100644 index 0000000000..351df334dc --- /dev/null +++ b/nano/lib/uniquer.hpp @@ -0,0 +1,80 @@ +#pragma once + +#include +#include +#include + +#include + +namespace nano +{ +template +class uniquer final +{ +public: + using key_type = Key; + using value_type = Value; + + std::shared_ptr unique (std::shared_ptr const & value) + { + if (value == nullptr) + { + return nullptr; + } + + // Types used as value need to provide full_hash() + Key hash = value->full_hash (); + + nano::lock_guard guard{ mutex }; + + if (cleanup_interval.elapsed ()) + { + cleanup (); + } + + auto & existing = values[hash]; + if (auto result = existing.lock ()) + { + return result; + } + else + { + existing = value; + } + + return value; + } + + std::size_t size () const + { + nano::lock_guard guard{ mutex }; + return values.size (); + } + + std::unique_ptr collect_container_info (std::string const & name) const + { + nano::lock_guard guard{ mutex }; + + auto composite = std::make_unique (name); + composite->add_component (std::make_unique (container_info{ "cache", values.size (), sizeof (Value) })); + return composite; + } + + static std::chrono::milliseconds constexpr cleanup_cutoff{ 500 }; + +private: + void cleanup () + { + debug_assert (!mutex.try_lock ()); + + std::erase_if (values, [] (auto const & item) { + return item.second.expired (); + }); + } + +private: + mutable nano::mutex mutex; + std::unordered_map> values; + nano::interval cleanup_interval{ cleanup_cutoff }; +}; +} \ No newline at end of file diff --git a/nano/node/node.cpp b/nano/node/node.cpp index e138f2e2f2..fb14c3c81c 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -180,7 +180,7 @@ nano::node::node (boost::asio::io_context & io_ctx_a, std::filesystem::path cons block_processor (*this, write_database_queue), online_reps (ledger, config), history{ config.network_params.voting }, - vote_uniquer (block_uniquer), + vote_uniquer{}, confirmation_height_processor (ledger, write_database_queue, config.conf_height_processor_batch_min_time, config.logging, logger, node_initialized_latch, flags.confirmation_height_processor_mode), vote_cache{ config.vote_cache, stats }, generator{ config, ledger, wallets, vote_processor, history, network, stats, /* non-final */ false }, @@ -565,8 +565,8 @@ std::unique_ptr nano::collect_container_info (no composite->add_component (collect_container_info (node.block_arrival, "block_arrival")); composite->add_component (collect_container_info (node.online_reps, "online_reps")); composite->add_component (collect_container_info (node.history, "history")); - composite->add_component (collect_container_info (node.block_uniquer, "block_uniquer")); - composite->add_component (collect_container_info (node.vote_uniquer, "vote_uniquer")); + composite->add_component (node.block_uniquer.collect_container_info ("block_uniquer")); + composite->add_component (node.vote_uniquer.collect_container_info ("vote_uniquer")); composite->add_component (collect_container_info (node.confirmation_height_processor, "confirmation_height_processor")); composite->add_component (collect_container_info (node.distributed_work, "distributed_work")); composite->add_component (collect_container_info (node.aggregator, "request_aggregator")); diff --git a/nano/rpc_test/rpc.cpp b/nano/rpc_test/rpc.cpp index 5416292cb0..f57569ac06 100644 --- a/nano/rpc_test/rpc.cpp +++ b/nano/rpc_test/rpc.cpp @@ -5810,7 +5810,7 @@ TEST (rpc, memory_stats) { auto response (wait_response (system, rpc_ctx, request)); - ASSERT_EQ (response.get_child ("node").get_child ("vote_uniquer").get_child ("votes").get ("count"), "1"); + ASSERT_EQ (response.get_child ("node").get_child ("vote_uniquer").get_child ("cache").get ("count"), "1"); } request.put ("type", "database"); diff --git a/nano/secure/common.cpp b/nano/secure/common.cpp index faf3b5d527..35b5ad68f9 100644 --- a/nano/secure/common.cpp +++ b/nano/secure/common.cpp @@ -430,69 +430,6 @@ nano::block_info::block_info (nano::account const & account_a, nano::amount cons { } -nano::vote_uniquer::vote_uniquer (nano::block_uniquer & uniquer_a) : - uniquer (uniquer_a) -{ -} - -std::shared_ptr nano::vote_uniquer::unique (std::shared_ptr const & vote_a) -{ - auto result = vote_a; - if (result != nullptr) - { - nano::block_hash key = vote_a->full_hash (); - nano::lock_guard lock{ mutex }; - auto & existing = votes[key]; - if (auto block_l = existing.lock ()) - { - result = block_l; - } - else - { - existing = vote_a; - } - - release_assert (std::numeric_limits::max () > votes.size ()); - for (auto i (0); i < cleanup_count && !votes.empty (); ++i) - { - auto random_offset = nano::random_pool::generate_word32 (0, static_cast (votes.size () - 1)); - - auto existing (std::next (votes.begin (), random_offset)); - if (existing == votes.end ()) - { - existing = votes.begin (); - } - if (existing != votes.end ()) - { - if (auto block_l = existing->second.lock ()) - { - // Still live - } - else - { - votes.erase (existing); - } - } - } - } - return result; -} - -size_t nano::vote_uniquer::size () -{ - nano::lock_guard lock{ mutex }; - return votes.size (); -} - -std::unique_ptr nano::collect_container_info (vote_uniquer & vote_uniquer, std::string const & name) -{ - auto count = vote_uniquer.size (); - auto sizeof_element = sizeof (vote_uniquer::value_type); - auto composite = std::make_unique (name); - composite->add_component (std::make_unique (container_info{ "votes", count, sizeof_element })); - return composite; -} - nano::wallet_id nano::random_wallet_id () { nano::wallet_id wallet_id; diff --git a/nano/secure/common.hpp b/nano/secure/common.hpp index 7b8fd35754..b14323edf5 100644 --- a/nano/secure/common.hpp +++ b/nano/secure/common.hpp @@ -244,27 +244,6 @@ namespace confirmation_height uint64_t const unbounded_cutoff{ 16384 }; } -/** - * This class serves to find and return unique variants of a vote in order to minimize memory usage - */ -class vote_uniquer final -{ -public: - using value_type = std::pair>; - - vote_uniquer (nano::block_uniquer &); - std::shared_ptr unique (std::shared_ptr const &); - size_t size (); - -private: - nano::block_uniquer & uniquer; - nano::mutex mutex{ mutex_identifier (mutexes::vote_uniquer) }; - std::unordered_map, value_type::second_type> votes; - static unsigned constexpr cleanup_count = 2; -}; - -std::unique_ptr collect_container_info (vote_uniquer & vote_uniquer, std::string const & name); - enum class vote_code { invalid, // Vote is not signed correctly diff --git a/nano/secure/vote.hpp b/nano/secure/vote.hpp index b20fcb3671..b8284c19dc 100644 --- a/nano/secure/vote.hpp +++ b/nano/secure/vote.hpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -77,4 +78,6 @@ class vote final // Vote timestamp uint64_t timestamp_m{ 0 }; }; + +using vote_uniquer = nano::uniquer; } \ No newline at end of file