From 45c7b4fe395f0428e2b017a0f82598098d83b196 Mon Sep 17 00:00:00 2001 From: RickiNano <81099017+RickiNano@users.noreply.github.com> Date: Sun, 12 Nov 2023 18:56:40 +0100 Subject: [PATCH] Set maximum unchecked blocks in config --- nano/core_test/block_store.cpp | 3 ++- nano/core_test/unchecked_map.cpp | 14 ++++++++------ nano/node/node.cpp | 2 +- nano/node/nodeconfig.cpp | 7 +++++++ nano/node/nodeconfig.hpp | 1 + nano/node/unchecked_map.cpp | 6 ++++-- nano/node/unchecked_map.hpp | 5 ++--- 7 files changed, 25 insertions(+), 13 deletions(-) diff --git a/nano/core_test/block_store.cpp b/nano/core_test/block_store.cpp index fd9a99e7d2..8c518cb7bc 100644 --- a/nano/core_test/block_store.cpp +++ b/nano/core_test/block_store.cpp @@ -436,7 +436,8 @@ TEST (block_store, empty_bootstrap) { nano::test::system system{}; nano::logger_mt logger; - nano::unchecked_map unchecked{ system.stats, false }; + unsigned max_unchecked_blocks = 65536; + nano::unchecked_map unchecked{ max_unchecked_blocks, system.stats, false }; size_t count = 0; unchecked.for_each ([&count] (nano::unchecked_key const & key, nano::unchecked_info const & info) { ++count; diff --git a/nano/core_test/unchecked_map.cpp b/nano/core_test/unchecked_map.cpp index b2012c4f5c..c1d75f574b 100644 --- a/nano/core_test/unchecked_map.cpp +++ b/nano/core_test/unchecked_map.cpp @@ -15,11 +15,13 @@ using namespace std::chrono_literals; namespace { +unsigned max_unchecked_blocks = 65536; + class context { public: context () : - unchecked{ stats, false } + unchecked{ max_unchecked_blocks, stats, false } { } nano::stats stats; @@ -55,7 +57,7 @@ TEST (unchecked_map, put_one) TEST (block_store, one_bootstrap) { nano::test::system system{}; - nano::unchecked_map unchecked{ system.stats, false }; + nano::unchecked_map unchecked{ max_unchecked_blocks, system.stats, false }; nano::block_builder builder; auto block1 = builder .send () @@ -88,7 +90,7 @@ TEST (block_store, one_bootstrap) TEST (unchecked, simple) { nano::test::system system{}; - nano::unchecked_map unchecked{ system.stats, false }; + nano::unchecked_map unchecked{ max_unchecked_blocks, system.stats, false }; nano::block_builder builder; auto block = builder .send () @@ -129,7 +131,7 @@ TEST (unchecked, multiple) // Don't test this in rocksdb mode GTEST_SKIP (); } - nano::unchecked_map unchecked{ system.stats, false }; + nano::unchecked_map unchecked{ max_unchecked_blocks, system.stats, false }; nano::block_builder builder; auto block = builder .send () @@ -159,7 +161,7 @@ TEST (unchecked, multiple) TEST (unchecked, double_put) { nano::test::system system{}; - nano::unchecked_map unchecked{ system.stats, false }; + nano::unchecked_map unchecked{ max_unchecked_blocks, system.stats, false }; nano::block_builder builder; auto block = builder .send () @@ -190,7 +192,7 @@ TEST (unchecked, double_put) TEST (unchecked, multiple_get) { nano::test::system system{}; - nano::unchecked_map unchecked{ system.stats, false }; + nano::unchecked_map unchecked{ max_unchecked_blocks, system.stats, false }; // Instantiates three blocks nano::block_builder builder; auto block1 = builder diff --git a/nano/node/node.cpp b/nano/node/node.cpp index fb14c3c81c..5c5cf85f6a 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -151,7 +151,7 @@ nano::node::node (boost::asio::io_context & io_ctx_a, std::filesystem::path cons logger (config_a.logging.min_time_between_log_output), store_impl (nano::make_store (logger, application_path_a, network_params.ledger, flags.read_only, true, config_a.rocksdb_config, config_a.diagnostics_config.txn_tracking, config_a.block_processor_batch_max_time, config_a.lmdb_config, config_a.backup_before_upgrade)), store (*store_impl), - unchecked{ stats, flags.disable_block_processor_unchecked_deletion }, + unchecked{ config.max_unchecked_blocks, stats, flags.disable_block_processor_unchecked_deletion }, wallets_store_impl (std::make_unique (application_path_a / "wallets.ldb", config_a.lmdb_config)), wallets_store (*wallets_store_impl), gap_cache (*this), diff --git a/nano/node/nodeconfig.cpp b/nano/node/nodeconfig.cpp index 2453c2c24d..8235d0e806 100644 --- a/nano/node/nodeconfig.cpp +++ b/nano/node/nodeconfig.cpp @@ -129,6 +129,7 @@ nano::error nano::node_config::serialize_toml (nano::tomlconfig & toml) const toml.put ("max_work_generate_multiplier", max_work_generate_multiplier, "Maximum allowed difficulty multiplier for work generation.\ntype:double,[1..]"); toml.put ("frontiers_confirmation", serialize_frontiers_confirmation (frontiers_confirmation), "Mode controlling frontier confirmation rate.\ntype:string,{auto,always,disabled}"); toml.put ("max_queued_requests", max_queued_requests, "Limit for number of queued confirmation requests for one channel, after which new requests are dropped until the queue drops below this value.\ntype:uint32"); + toml.put ("max_unchecked_blocks", max_unchecked_blocks, "Maximum number of unchecked blocks to store in memory. Defaults to 65536. \ntype:uint64,[1..11]"); toml.put ("rep_crawler_weight_minimum", rep_crawler_weight_minimum.to_string_dec (), "Rep crawler minimum weight, if this is less than minimum principal weight then this is taken as the minimum weight a rep must have to be tracked. If you want to track all reps set this to 0. If you do not want this to influence anything then set it to max value. This is only useful for debugging or for people who really know what they are doing.\ntype:string,amount,raw"); toml.put ("backlog_scan_batch_size", backlog_scan_batch_size, "Number of accounts per second to process when doing backlog population scan. Increasing this value will help unconfirmed frontiers get into election prioritization queue faster, however it will also increase resource usage. \ntype:uint"); toml.put ("backlog_scan_frequency", backlog_scan_frequency, "Backlog scan divides the scan into smaller batches, number of which is controlled by this value. Higher frequency helps to utilize resources more uniformly, however it also introduces more overhead. The resulting number of accounts per single batch is `backlog_scan_batch_size / backlog_scan_frequency` \ntype:uint"); @@ -426,6 +427,8 @@ nano::error nano::node_config::deserialize_toml (nano::tomlconfig & toml) toml.get ("max_queued_requests", max_queued_requests); + toml.get ("max_unchecked_blocks", max_unchecked_blocks); + auto rep_crawler_weight_minimum_l (rep_crawler_weight_minimum.to_string_dec ()); if (toml.has_key ("rep_crawler_weight_minimum")) { @@ -473,6 +476,10 @@ nano::error nano::node_config::deserialize_toml (nano::tomlconfig & toml) { toml.get_error ().set ("active_elections_size must be greater than 250"); } + if (max_unchecked_blocks < 1) + { + toml.get_error ().set ("max_unchecked_blocks must be greater than 0"); + } if (bandwidth_limit > std::numeric_limits::max ()) { toml.get_error ().set ("bandwidth_limit unbounded = 0, default = 10485760, max = 18446744073709551615"); diff --git a/nano/node/nodeconfig.hpp b/nano/node/nodeconfig.hpp index 0cc4116270..7b7547d654 100644 --- a/nano/node/nodeconfig.hpp +++ b/nano/node/nodeconfig.hpp @@ -116,6 +116,7 @@ class node_config bool backup_before_upgrade{ false }; double max_work_generate_multiplier{ 64. }; uint32_t max_queued_requests{ 512 }; + unsigned max_unchecked_blocks{ 65536 }; std::chrono::seconds max_pruning_age{ !network_params.network.is_beta_network () ? std::chrono::seconds (24 * 60 * 60) : std::chrono::seconds (5 * 60) }; // 1 day; 5 minutes for beta network uint64_t max_pruning_depth{ 0 }; nano::rocksdb_config rocksdb_config; diff --git a/nano/node/unchecked_map.cpp b/nano/node/unchecked_map.cpp index 45d3b49563..9c9ccd53f1 100644 --- a/nano/node/unchecked_map.cpp +++ b/nano/node/unchecked_map.cpp @@ -5,7 +5,8 @@ #include #include -nano::unchecked_map::unchecked_map (nano::stats & stats, bool const & disable_delete) : +nano::unchecked_map::unchecked_map (unsigned & max_unchecked_blocks, nano::stats & stats, bool const & disable_delete) : + max_unchecked_blocks{ max_unchecked_blocks }, stats{ stats }, disable_delete{ disable_delete }, thread{ [this] () { run (); } } @@ -23,7 +24,8 @@ void nano::unchecked_map::put (nano::hash_or_account const & dependency, nano::u nano::lock_guard lock{ entries_mutex }; nano::unchecked_key key{ dependency, info.block->hash () }; entries.get ().insert ({ key, info }); - if (entries.size () > mem_block_count_max) + + if (entries.size () > max_unchecked_blocks) { entries.get ().pop_front (); } diff --git a/nano/node/unchecked_map.hpp b/nano/node/unchecked_map.hpp index b76e8ccb44..c733ba20c8 100644 --- a/nano/node/unchecked_map.hpp +++ b/nano/node/unchecked_map.hpp @@ -22,7 +22,7 @@ class stats; class unchecked_map { public: - unchecked_map (nano::stats &, bool const & do_delete); + unchecked_map (unsigned & max_unchecked_blocks, nano::stats &, bool const & do_delete); ~unchecked_map (); void put (nano::hash_or_account const & dependency, nano::unchecked_info const & info); @@ -62,11 +62,10 @@ class unchecked_map nano::condition_variable condition; nano::mutex mutex; std::thread thread; + unsigned max_unchecked_blocks; void process_queries (decltype (buffer) const & back_buffer); - static std::size_t constexpr mem_block_count_max = 64 * 1024; - private: struct entry {