From d2d01fe2bc29bec8a405c66ea35796443af687ef Mon Sep 17 00:00:00 2001 From: Colin LeMahieu Date: Mon, 25 Sep 2023 21:44:16 +0100 Subject: [PATCH 1/7] Moving block successors to their own table. --- nano/core_test/block_store.cpp | 51 ++-------- nano/core_test/ledger.cpp | 4 +- nano/lib/blocks.cpp | 9 +- nano/lib/blocks.hpp | 5 +- nano/nano_node/entry.cpp | 2 +- nano/node/bootstrap/bootstrap_bulk_pull.cpp | 4 +- nano/node/bootstrap/bootstrap_server.cpp | 5 +- nano/node/confirmation_height_bounded.cpp | 15 +-- nano/node/json_handler.cpp | 10 +- nano/node/node.cpp | 6 ++ nano/node/node.hpp | 1 + nano/node/request_aggregator.cpp | 2 +- nano/node/scheduler/priority.cpp | 2 +- nano/secure/common.cpp | 8 +- nano/secure/ledger.cpp | 36 ++++--- nano/store/CMakeLists.txt | 6 ++ nano/store/block.hpp | 2 - nano/store/component.cpp | 5 +- nano/store/component.hpp | 5 +- nano/store/lmdb/block.cpp | 102 -------------------- nano/store/lmdb/block.hpp | 3 - nano/store/lmdb/lmdb.cpp | 7 +- nano/store/lmdb/lmdb.hpp | 3 + nano/store/lmdb/successor.cpp | 35 +++++++ nano/store/lmdb/successor.hpp | 29 ++++++ nano/store/rocksdb/block.cpp | 102 -------------------- nano/store/rocksdb/block.hpp | 3 - nano/store/rocksdb/rocksdb.cpp | 4 +- nano/store/rocksdb/rocksdb.hpp | 3 + nano/store/rocksdb/successor.cpp | 32 ++++++ nano/store/rocksdb/successor.hpp | 21 ++++ nano/store/successor.cpp | 1 + nano/store/successor.hpp | 22 +++++ nano/store/tables.hpp | 1 + 34 files changed, 238 insertions(+), 308 deletions(-) create mode 100644 nano/store/lmdb/successor.cpp create mode 100644 nano/store/lmdb/successor.hpp create mode 100644 nano/store/rocksdb/successor.cpp create mode 100644 nano/store/rocksdb/successor.hpp create mode 100644 nano/store/successor.cpp create mode 100644 nano/store/successor.hpp diff --git a/nano/core_test/block_store.cpp b/nano/core_test/block_store.cpp index 573f412eb4..13f7fd9b06 100644 --- a/nano/core_test/block_store.cpp +++ b/nano/core_test/block_store.cpp @@ -85,7 +85,6 @@ TEST (block_store, sideband_serialization) sideband1.account = 1; sideband1.balance = 2; sideband1.height = 3; - sideband1.successor = 4; sideband1.timestamp = 5; std::vector vector; { @@ -98,7 +97,6 @@ TEST (block_store, sideband_serialization) ASSERT_EQ (sideband1.account, sideband2.account); ASSERT_EQ (sideband1.balance, sideband2.balance); ASSERT_EQ (sideband1.height, sideband2.height); - ASSERT_EQ (sideband1.successor, sideband2.successor); ASSERT_EQ (sideband1.timestamp, sideband2.timestamp); } @@ -137,47 +135,14 @@ TEST (block_store, clear_successor) { nano::logger_mt logger; auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants); - ASSERT_TRUE (!store->init_error ()); - nano::block_builder builder; - auto block1 = builder - .open () - .source (0) - .representative (1) - .account (0) - .sign (nano::keypair ().prv, 0) - .work (0) - .build (); - block1->sideband_set ({}); - auto transaction (store->tx_begin_write ()); - store->block.put (transaction, block1->hash (), *block1); - auto block2 = builder - .open () - .source (0) - .representative (2) - .account (0) - .sign (nano::keypair ().prv, 0) - .work (0) - .build (); - block2->sideband_set ({}); - store->block.put (transaction, block2->hash (), *block2); - auto block2_store (store->block.get (transaction, block1->hash ())); - ASSERT_NE (nullptr, block2_store); - ASSERT_EQ (0, block2_store->sideband ().successor.number ()); - auto modified_sideband = block2_store->sideband (); - modified_sideband.successor = block2->hash (); - block1->sideband_set (modified_sideband); - store->block.put (transaction, block1->hash (), *block1); - { - auto block1_store (store->block.get (transaction, block1->hash ())); - ASSERT_NE (nullptr, block1_store); - ASSERT_EQ (block2->hash (), block1_store->sideband ().successor); - } - store->block.successor_clear (transaction, block1->hash ()); - { - auto block1_store (store->block.get (transaction, block1->hash ())); - ASSERT_NE (nullptr, block1_store); - ASSERT_EQ (0, block1_store->sideband ().successor.number ()); - } + nano::block_hash one{ 1 }; + nano::block_hash two{ 2 }; + auto tx = store->tx_begin_write (); + ASSERT_EQ (0, store->successor.get (tx, one)); + store->successor.put (tx, one, two); + ASSERT_EQ (two, store->successor.get (tx, one)); + store->successor.del (tx, one); + ASSERT_EQ (0, store->successor.get (tx, one)); } TEST (block_store, add_nonempty_block) diff --git a/nano/core_test/ledger.cpp b/nano/core_test/ledger.cpp index b2d08ba9cf..4b772a07ac 100644 --- a/nano/core_test/ledger.cpp +++ b/nano/core_test/ledger.cpp @@ -266,7 +266,7 @@ TEST (ledger, process_receive) ASSERT_EQ (nano::dev::constants.genesis_amount - 25, ledger.account_balance (transaction, key2.pub)); ASSERT_EQ (nano::dev::constants.genesis_amount - 25, ledger.weight (key3.pub)); ASSERT_FALSE (ledger.rollback (transaction, hash4)); - ASSERT_TRUE (store.block.successor (transaction, hash2).is_zero ()); + ASSERT_TRUE (store.successor.get (transaction, hash2).is_zero ()); ASSERT_EQ (key2.pub, store.frontier.get (transaction, hash2)); ASSERT_TRUE (store.frontier.get (transaction, hash4).is_zero ()); ASSERT_EQ (25, ledger.account_balance (transaction, nano::dev::genesis_key.pub)); @@ -3109,7 +3109,7 @@ TEST (ledger, state_rollback_send) ASSERT_EQ (nano::dev::constants.genesis_amount, ledger.account_balance (transaction, nano::dev::genesis->account ())); ASSERT_EQ (nano::dev::constants.genesis_amount, ledger.weight (nano::dev::genesis->account ())); ASSERT_FALSE (store.pending.exists (transaction, nano::pending_key (nano::dev::genesis->account (), send1->hash ()))); - ASSERT_TRUE (store.block.successor (transaction, nano::dev::genesis->hash ()).is_zero ()); + ASSERT_TRUE (store.successor.get (transaction, nano::dev::genesis->hash ()).is_zero ()); ASSERT_EQ (store.account.count (transaction), ledger.cache.account_count); } diff --git a/nano/lib/blocks.cpp b/nano/lib/blocks.cpp index fab4c2466b..47c12d3922 100644 --- a/nano/lib/blocks.cpp +++ b/nano/lib/blocks.cpp @@ -1754,8 +1754,7 @@ std::string nano::state_subtype (nano::block_details const details_a) } } -nano::block_sideband::block_sideband (nano::account const & account_a, nano::block_hash const & successor_a, nano::amount const & balance_a, uint64_t const height_a, nano::seconds_t const timestamp_a, nano::block_details const & details_a, nano::epoch const source_epoch_a) : - successor (successor_a), +nano::block_sideband::block_sideband (nano::account const & account_a, nano::amount const & balance_a, uint64_t const height_a, nano::seconds_t const timestamp_a, nano::block_details const & details_a, nano::epoch const source_epoch_a) : account (account_a), balance (balance_a), height (height_a), @@ -1765,8 +1764,7 @@ nano::block_sideband::block_sideband (nano::account const & account_a, nano::blo { } -nano::block_sideband::block_sideband (nano::account const & account_a, nano::block_hash const & successor_a, nano::amount const & balance_a, uint64_t const height_a, nano::seconds_t const timestamp_a, nano::epoch const epoch_a, bool const is_send, bool const is_receive, bool const is_epoch, nano::epoch const source_epoch_a) : - successor (successor_a), +nano::block_sideband::block_sideband (nano::account const & account_a, nano::amount const & balance_a, uint64_t const height_a, nano::seconds_t const timestamp_a, nano::epoch const epoch_a, bool const is_send, bool const is_receive, bool const is_epoch, nano::epoch const source_epoch_a) : account (account_a), balance (balance_a), height (height_a), @@ -1779,7 +1777,6 @@ nano::block_sideband::block_sideband (nano::account const & account_a, nano::blo size_t nano::block_sideband::size (nano::block_type type_a) { size_t result (0); - result += sizeof (successor); if (type_a != nano::block_type::state && type_a != nano::block_type::open) { result += sizeof (account); @@ -1803,7 +1800,6 @@ size_t nano::block_sideband::size (nano::block_type type_a) void nano::block_sideband::serialize (nano::stream & stream_a, nano::block_type type_a) const { - nano::write (stream_a, successor.bytes); if (type_a != nano::block_type::state && type_a != nano::block_type::open) { nano::write (stream_a, account.bytes); @@ -1829,7 +1825,6 @@ bool nano::block_sideband::deserialize (nano::stream & stream_a, nano::block_typ bool result (false); try { - nano::read (stream_a, successor.bytes); if (type_a != nano::block_type::state && type_a != nano::block_type::open) { nano::read (stream_a, account.bytes); diff --git a/nano/lib/blocks.hpp b/nano/lib/blocks.hpp index abfb6521ca..c441970de2 100644 --- a/nano/lib/blocks.hpp +++ b/nano/lib/blocks.hpp @@ -59,12 +59,11 @@ class block_sideband final { public: block_sideband () = default; - block_sideband (nano::account const &, nano::block_hash const &, nano::amount const &, uint64_t const, nano::seconds_t const local_timestamp, nano::block_details const &, nano::epoch const source_epoch_a); - block_sideband (nano::account const &, nano::block_hash const &, nano::amount const &, uint64_t const, nano::seconds_t const local_timestamp, nano::epoch const epoch_a, bool const is_send, bool const is_receive, bool const is_epoch, nano::epoch const source_epoch_a); + block_sideband (nano::account const &, nano::amount const &, uint64_t const, nano::seconds_t const local_timestamp, nano::block_details const &, nano::epoch const source_epoch_a); + block_sideband (nano::account const &, nano::amount const &, uint64_t const, nano::seconds_t const local_timestamp, nano::epoch const epoch_a, bool const is_send, bool const is_receive, bool const is_epoch, nano::epoch const source_epoch_a); void serialize (nano::stream &, nano::block_type) const; bool deserialize (nano::stream &, nano::block_type); static size_t size (nano::block_type); - nano::block_hash successor{ 0 }; nano::account account{}; nano::amount balance{ 0 }; uint64_t height{ 0 }; diff --git a/nano/nano_node/entry.cpp b/nano/nano_node/entry.cpp index ab21ab03c5..7988c4cf9d 100644 --- a/nano/nano_node/entry.cpp +++ b/nano/nano_node/entry.cpp @@ -1600,7 +1600,7 @@ int main (int argc, char * const * argv) calculated_representative = block->representative (); } // Retrieving successor block hash - hash = node->store.block.successor (transaction, hash); + hash = node->store.successor.get (transaction, hash); // Retrieving block data if (!hash.is_zero ()) { diff --git a/nano/node/bootstrap/bootstrap_bulk_pull.cpp b/nano/node/bootstrap/bootstrap_bulk_pull.cpp index 5c0daa4602..7a07e8008f 100644 --- a/nano/node/bootstrap/bootstrap_bulk_pull.cpp +++ b/nano/node/bootstrap/bootstrap_bulk_pull.cpp @@ -393,7 +393,7 @@ void nano::bulk_pull_server::set_current_end () node->logger.try_log (boost::str (boost::format ("Bulk pull request for block hash: %1%") % request->start.to_string ())); } - current = ascending () ? node->store.block.successor (transaction, request->start.as_block_hash ()) : request->start.as_block_hash (); + current = ascending () ? node->store.successor.get (transaction, request->start.as_block_hash ()) : request->start.as_block_hash (); include_start = true; } else @@ -514,7 +514,7 @@ std::shared_ptr nano::bulk_pull_server::get_next () result = node->block (current); if (result != nullptr && set_current_to_end == false) { - auto next = ascending () ? result->sideband ().successor : result->previous (); + auto next = ascending () ? node->successor (current) : result->previous (); if (!next.is_zero ()) { current = next; diff --git a/nano/node/bootstrap/bootstrap_server.cpp b/nano/node/bootstrap/bootstrap_server.cpp index 5bfc959220..7a43ef7c01 100644 --- a/nano/node/bootstrap/bootstrap_server.cpp +++ b/nano/node/bootstrap/bootstrap_server.cpp @@ -5,6 +5,7 @@ #include #include #include +#include // TODO: Make threads configurable nano::bootstrap_server::bootstrap_server (nano::store::component & store_a, nano::ledger & ledger_a, nano::network_constants const & network_constants_a, nano::stats & stats_a) : @@ -244,9 +245,7 @@ std::vector> nano::bootstrap_server::prepare_blocks while (current && result.size () < count) { result.push_back (current); - - auto successor = current->sideband ().successor; - current = store.block.get (transaction, successor); + current = store.block.get (transaction, store.successor.get (transaction, current->hash ())); } } return result; diff --git a/nano/node/confirmation_height_bounded.cpp b/nano/node/confirmation_height_bounded.cpp index a2301f891d..bbfaf809ca 100644 --- a/nano/node/confirmation_height_bounded.cpp +++ b/nano/node/confirmation_height_bounded.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include @@ -233,7 +234,7 @@ nano::block_hash nano::confirmation_height_bounded::get_least_unconfirmed_hash_f { auto block (ledger.store.block.get (transaction_a, confirmation_height_info_a.frontier)); release_assert (block != nullptr); - least_unconfirmed_hash = block->sideband ().successor; + least_unconfirmed_hash = ledger.store.successor.get (transaction_a, confirmation_height_info_a.frontier); block_height_a = block->sideband ().height + 1; } } @@ -260,6 +261,7 @@ bool nano::confirmation_height_bounded::iterate (store::read_transaction const & // Once a receive is cemented, we can cement all blocks above it until the next receive, so store those details for later. ++num_blocks; auto block = ledger.store.block.get (transaction_a, hash); + auto successor = ledger.store.successor.get (transaction_a, hash); auto source (block->source ()); if (source.is_zero ()) { @@ -270,9 +272,8 @@ bool nano::confirmation_height_bounded::iterate (store::read_transaction const & { hit_receive = true; reached_target = true; - auto const & sideband (block->sideband ()); - auto next = !sideband.successor.is_zero () && sideband.successor != top_level_hash_a ? boost::optional (sideband.successor) : boost::none; - receive_source_pairs_a.push_back ({ receive_chain_details{ account_a, sideband.height, hash, top_level_hash_a, next, bottom_height_a, bottom_hash_a }, source }); + auto next = !successor.is_zero () && successor != top_level_hash_a ? boost::optional (successor) : boost::none; + receive_source_pairs_a.push_back ({ receive_chain_details{ account_a, block->sideband ().height, hash, top_level_hash_a, next, bottom_height_a, bottom_hash_a }, source }); // Store a checkpoint every max_items so that we can always traverse a long number of accounts to genesis if (receive_source_pairs_a.size () % max_items == 0) { @@ -289,7 +290,7 @@ bool nano::confirmation_height_bounded::iterate (store::read_transaction const & } else { - hash = block->sideband ().successor; + hash = successor; } } @@ -419,7 +420,7 @@ void nano::confirmation_height_bounded::cement_blocks (nano::write_guard & scope else { auto block = ledger.store.block.get (transaction, confirmation_height_info.frontier); - new_cemented_frontier = block->sideband ().successor; + new_cemented_frontier = ledger.store.successor.get (transaction, confirmation_height_info.frontier); num_blocks_confirmed = pending.top_height - confirmation_height_info.height; start_height = confirmation_height_info.height + 1; } @@ -488,7 +489,7 @@ void nano::confirmation_height_bounded::cement_blocks (nano::write_guard & scope // Get the next block in the chain until we have reached the final desired one if (!last_iteration) { - new_cemented_frontier = block->sideband ().successor; + new_cemented_frontier = ledger.store.successor.get (transaction, new_cemented_frontier); block = ledger.store.block.get (transaction, new_cemented_frontier); } else diff --git a/nano/node/json_handler.cpp b/nano/node/json_handler.cpp index 3e6683da74..8f49dc64fa 100644 --- a/nano/node/json_handler.cpp +++ b/nano/node/json_handler.cpp @@ -1160,7 +1160,7 @@ void nano::json_handler::block_info () response_l.put ("balance", balance.convert_to ()); response_l.put ("height", std::to_string (block->sideband ().height)); response_l.put ("local_timestamp", std::to_string (block->sideband ().timestamp)); - response_l.put ("successor", block->sideband ().successor.to_string ()); + response_l.put ("successor", node.store.successor.get (transaction, hash).to_string ()); auto confirmed (node.ledger.block_confirmed (transaction, hash)); response_l.put ("confirmed", confirmed); @@ -1320,7 +1320,7 @@ void nano::json_handler::blocks_info () entry.put ("balance", balance.convert_to ()); entry.put ("height", std::to_string (block->sideband ().height)); entry.put ("local_timestamp", std::to_string (block->sideband ().timestamp)); - entry.put ("successor", block->sideband ().successor.to_string ()); + entry.put ("successor", node.store.successor.get (transaction, hash).to_string ()); auto confirmed (node.ledger.block_confirmed (transaction, hash)); entry.put ("confirmed", confirmed); @@ -1959,7 +1959,7 @@ void nano::json_handler::chain (bool successors) entry.put ("", hash.to_string ()); blocks.push_back (std::make_pair ("", entry)); } - hash = successors ? node.store.block.successor (transaction, hash) : block_l->previous (); + hash = successors ? node.store.successor.get (transaction, hash) : block_l->previous (); } else { @@ -2671,7 +2671,7 @@ void nano::json_handler::account_history () --count; } } - hash = reverse ? node.store.block.successor (transaction, hash) : block->previous (); + hash = reverse ? node.store.successor.get (transaction, hash) : block->previous (); block = node.store.block.get (transaction, hash); } response_l.add_child ("history", history); @@ -3706,7 +3706,7 @@ void nano::json_handler::republish () } } } - hash = node.store.block.successor (transaction, hash); + hash = node.store.successor.get (transaction, hash); } node.network.flood_block_many (std::move (republish_bundle), nullptr, 25); response_l.put ("success", ""); // obsolete diff --git a/nano/node/node.cpp b/nano/node/node.cpp index 049e940c69..04d1e193d8 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -773,6 +773,12 @@ std::shared_ptr nano::node::block (nano::block_hash const & hash_a) return store.block.get (transaction, hash_a); } +nano::block_hash nano::node::successor (nano::block_hash const & hash) +{ + auto tx = store.tx_begin_read (); + return store.successor.get (tx, hash); +} + std::pair nano::node::balance_pending (nano::account const & account_a, bool only_confirmed_a) { std::pair result; diff --git a/nano/node/node.hpp b/nano/node/node.hpp index 870562dc55..d4167977d5 100644 --- a/nano/node/node.hpp +++ b/nano/node/node.hpp @@ -95,6 +95,7 @@ class node final : public std::enable_shared_from_this void process_local_async (std::shared_ptr const &); void keepalive_preconfigured (std::vector const &); std::shared_ptr block (nano::block_hash const &); + nano::block_hash successor (nano::block_hash const & hash); std::pair balance_pending (nano::account const &, bool only_confirmed); nano::uint128_t weight (nano::account const &); nano::block_hash rep_block (nano::account const &); diff --git a/nano/node/request_aggregator.cpp b/nano/node/request_aggregator.cpp index cb84ba6045..3192458983 100644 --- a/nano/node/request_aggregator.cpp +++ b/nano/node/request_aggregator.cpp @@ -237,7 +237,7 @@ std::pair>, std::vectorblock_count) { debug_assert (conf_info.frontier != info->head); - auto hash = conf_info.height == 0 ? info->open_block : node.store.block.successor (transaction, conf_info.frontier); + auto hash = conf_info.height == 0 ? info->open_block : node.store.successor.get (transaction, conf_info.frontier); auto block = node.store.block.get (transaction, hash); debug_assert (block != nullptr); if (node.ledger.dependents_confirmed (transaction, *block)) diff --git a/nano/secure/common.cpp b/nano/secure/common.cpp index c26b32f893..c6bc56ab92 100644 --- a/nano/secure/common.cpp +++ b/nano/secure/common.cpp @@ -130,10 +130,10 @@ nano::ledger_constants::ledger_constants (nano::work_thresholds & work, nano::ne : network_a == nano::networks::nano_test_network ? nano_test_final_votes_canary_height : nano_live_final_votes_canary_height) { - nano_beta_genesis->sideband_set (nano::block_sideband (nano_beta_genesis->account (), 0, std::numeric_limits::max (), 1, nano::seconds_since_epoch (), nano::epoch::epoch_0, false, false, false, nano::epoch::epoch_0)); - nano_dev_genesis->sideband_set (nano::block_sideband (nano_dev_genesis->account (), 0, std::numeric_limits::max (), 1, nano::seconds_since_epoch (), nano::epoch::epoch_0, false, false, false, nano::epoch::epoch_0)); - nano_live_genesis->sideband_set (nano::block_sideband (nano_live_genesis->account (), 0, std::numeric_limits::max (), 1, nano::seconds_since_epoch (), nano::epoch::epoch_0, false, false, false, nano::epoch::epoch_0)); - nano_test_genesis->sideband_set (nano::block_sideband (nano_test_genesis->account (), 0, std::numeric_limits::max (), 1, nano::seconds_since_epoch (), nano::epoch::epoch_0, false, false, false, nano::epoch::epoch_0)); + nano_beta_genesis->sideband_set (nano::block_sideband (nano_beta_genesis->account (), std::numeric_limits::max (), 1, nano::seconds_since_epoch (), nano::epoch::epoch_0, false, false, false, nano::epoch::epoch_0)); + nano_dev_genesis->sideband_set (nano::block_sideband (nano_dev_genesis->account (), std::numeric_limits::max (), 1, nano::seconds_since_epoch (), nano::epoch::epoch_0, false, false, false, nano::epoch::epoch_0)); + nano_live_genesis->sideband_set (nano::block_sideband (nano_live_genesis->account (), std::numeric_limits::max (), 1, nano::seconds_since_epoch (), nano::epoch::epoch_0, false, false, false, nano::epoch::epoch_0)); + nano_test_genesis->sideband_set (nano::block_sideband (nano_test_genesis->account (), std::numeric_limits::max (), 1, nano::seconds_since_epoch (), nano::epoch::epoch_0, false, false, false, nano::epoch::epoch_0)); nano::link epoch_link_v1; char const * epoch_message_v1 ("epoch v1 block"); diff --git a/nano/secure/ledger.cpp b/nano/secure/ledger.cpp index a42d29401a..eec0e2fc1a 100644 --- a/nano/secure/ledger.cpp +++ b/nano/secure/ledger.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -56,7 +57,7 @@ class rollback_visitor : public nano::block_visitor ledger.store.block.del (transaction, hash); ledger.store.frontier.del (transaction, hash); ledger.store.frontier.put (transaction, block_a.hashables.previous, pending.source); - ledger.store.block.successor_clear (transaction, block_a.hashables.previous); + ledger.store.successor.del (transaction, block_a.hashables.previous); ledger.stats.inc (nano::stat::type::rollback, nano::stat::detail::send); } } @@ -77,7 +78,7 @@ class rollback_visitor : public nano::block_visitor ledger.store.pending.put (transaction, nano::pending_key (destination_account, block_a.hashables.source), { source_account, amount, nano::epoch::epoch_0 }); ledger.store.frontier.del (transaction, hash); ledger.store.frontier.put (transaction, block_a.hashables.previous, destination_account); - ledger.store.block.successor_clear (transaction, block_a.hashables.previous); + ledger.store.successor.del (transaction, block_a.hashables.previous); ledger.stats.inc (nano::stat::type::rollback, nano::stat::detail::receive); } void open_block (nano::open_block const & block_a) override @@ -113,7 +114,7 @@ class rollback_visitor : public nano::block_visitor ledger.update_account (transaction, account, *info, new_info); ledger.store.frontier.del (transaction, hash); ledger.store.frontier.put (transaction, block_a.hashables.previous, account); - ledger.store.block.successor_clear (transaction, block_a.hashables.previous); + ledger.store.successor.del (transaction, block_a.hashables.previous); ledger.stats.inc (nano::stat::type::rollback, nano::stat::detail::change); } void state_block (nano::state_block const & block_a) override @@ -172,7 +173,7 @@ class rollback_visitor : public nano::block_visitor auto previous (ledger.store.block.get (transaction, block_a.hashables.previous)); if (previous != nullptr) { - ledger.store.block.successor_clear (transaction, block_a.hashables.previous); + ledger.store.successor.del (transaction, block_a.hashables.previous); if (previous->type () < nano::block_type::state) { ledger.store.frontier.put (transaction, block_a.hashables.previous, block_a.hashables.account); @@ -343,8 +344,12 @@ void ledger_processor::state_block_impl (nano::state_block & block_a) if (result.code == nano::process_result::progress) { ledger.stats.inc (nano::stat::type::ledger, nano::stat::detail::state_block); - block_a.sideband_set (nano::block_sideband (block_a.hashables.account /* unused */, 0, 0 /* unused */, info.block_count + 1, nano::seconds_since_epoch (), block_details, source_epoch)); + block_a.sideband_set (nano::block_sideband (block_a.hashables.account /* unused */, 0 /* unused */, info.block_count + 1, nano::seconds_since_epoch (), block_details, source_epoch)); ledger.store.block.put (transaction, hash, block_a); + if (!block_a.hashables.previous.is_zero ()) + { + ledger.store.successor.put (transaction, block_a.hashables.previous, hash); + } if (!info.head.is_zero ()) { @@ -436,8 +441,12 @@ void ledger_processor::epoch_block_impl (nano::state_block & block_a) if (result.code == nano::process_result::progress) { ledger.stats.inc (nano::stat::type::ledger, nano::stat::detail::epoch_block); - block_a.sideband_set (nano::block_sideband (block_a.hashables.account /* unused */, 0, 0 /* unused */, info.block_count + 1, nano::seconds_since_epoch (), block_details, nano::epoch::epoch_0 /* unused */)); + block_a.sideband_set (nano::block_sideband (block_a.hashables.account /* unused */, 0 /* unused */, info.block_count + 1, nano::seconds_since_epoch (), block_details, nano::epoch::epoch_0 /* unused */)); ledger.store.block.put (transaction, hash, block_a); + if (!block_a.hashables.previous.is_zero ()) + { + ledger.store.successor.put (transaction, block_a.hashables.previous, hash); + } nano::account_info new_info (hash, block_a.representative (), info.open_block.is_zero () ? hash : info.open_block, info.balance, nano::seconds_since_epoch (), info.block_count + 1, epoch); ledger.update_account (transaction, block_a.hashables.account, info, new_info); if (!ledger.store.frontier.get (transaction, info.head).is_zero ()) @@ -482,8 +491,9 @@ void ledger_processor::change_block (nano::change_block & block_a) if (result.code == nano::process_result::progress) { debug_assert (!validate_message (account, hash, block_a.signature)); - block_a.sideband_set (nano::block_sideband (account, 0, info->balance, info->block_count + 1, nano::seconds_since_epoch (), block_details, nano::epoch::epoch_0 /* unused */)); + block_a.sideband_set (nano::block_sideband (account, info->balance, info->block_count + 1, nano::seconds_since_epoch (), block_details, nano::epoch::epoch_0 /* unused */)); ledger.store.block.put (transaction, hash, block_a); + ledger.store.successor.put (transaction, block_a.hashables.previous, hash); auto balance (ledger.balance (transaction, block_a.hashables.previous)); ledger.cache.rep_weights.representation_add_dual (block_a.representative (), balance, info->representative, 0 - balance); nano::account_info new_info (hash, block_a.representative (), info->open_block, info->balance, nano::seconds_since_epoch (), info->block_count + 1, nano::epoch::epoch_0); @@ -533,8 +543,9 @@ void ledger_processor::send_block (nano::send_block & block_a) { auto amount (info->balance.number () - block_a.hashables.balance.number ()); ledger.cache.rep_weights.representation_add (info->representative, 0 - amount); - block_a.sideband_set (nano::block_sideband (account, 0, block_a.hashables.balance /* unused */, info->block_count + 1, nano::seconds_since_epoch (), block_details, nano::epoch::epoch_0 /* unused */)); + block_a.sideband_set (nano::block_sideband (account, block_a.hashables.balance /* unused */, info->block_count + 1, nano::seconds_since_epoch (), block_details, nano::epoch::epoch_0 /* unused */)); ledger.store.block.put (transaction, hash, block_a); + ledger.store.successor.put (transaction, block_a.hashables.previous, hash); nano::account_info new_info (hash, info->representative, info->open_block, block_a.hashables.balance, nano::seconds_since_epoch (), info->block_count + 1, nano::epoch::epoch_0); ledger.update_account (transaction, account, *info, new_info); ledger.store.pending.put (transaction, nano::pending_key (block_a.hashables.destination, hash), { account, amount, nano::epoch::epoch_0 }); @@ -601,8 +612,9 @@ void ledger_processor::receive_block (nano::receive_block & block_a) } #endif ledger.store.pending.del (transaction, key); - block_a.sideband_set (nano::block_sideband (account, 0, new_balance, info->block_count + 1, nano::seconds_since_epoch (), block_details, nano::epoch::epoch_0 /* unused */)); + block_a.sideband_set (nano::block_sideband (account, new_balance, info->block_count + 1, nano::seconds_since_epoch (), block_details, nano::epoch::epoch_0 /* unused */)); ledger.store.block.put (transaction, hash, block_a); + ledger.store.successor.put (transaction, block_a.hashables.previous, hash); nano::account_info new_info (hash, info->representative, info->open_block, new_balance, nano::seconds_since_epoch (), info->block_count + 1, nano::epoch::epoch_0); ledger.update_account (transaction, account, *info, new_info); ledger.cache.rep_weights.representation_add (info->representative, pending.amount.number ()); @@ -667,7 +679,7 @@ void ledger_processor::open_block (nano::open_block & block_a) } #endif ledger.store.pending.del (transaction, key); - block_a.sideband_set (nano::block_sideband (block_a.hashables.account, 0, pending.amount, 1, nano::seconds_since_epoch (), block_details, nano::epoch::epoch_0 /* unused */)); + block_a.sideband_set (nano::block_sideband (block_a.hashables.account, pending.amount, 1, nano::seconds_since_epoch (), block_details, nano::epoch::epoch_0 /* unused */)); ledger.store.block.put (transaction, hash, block_a); nano::account_info new_info (hash, block_a.representative (), hash, pending.amount.number (), nano::seconds_since_epoch (), 1, nano::epoch::epoch_0); ledger.update_account (transaction, block_a.hashables.account, info, new_info); @@ -1434,7 +1446,7 @@ std::shared_ptr nano::ledger::successor (store::transaction const & if (get_from_previous) { - successor = store.block.successor (transaction_a, root_a.previous ()); + successor = store.successor.get (transaction_a, root_a.previous ()); } std::shared_ptr result; if (!successor.is_zero ()) @@ -1450,7 +1462,7 @@ std::shared_ptr nano::ledger::forked_block (store::transaction cons debug_assert (!store.block.exists (transaction_a, block_a.hash ())); auto root (block_a.root ()); debug_assert (store.block.exists (transaction_a, root.as_block_hash ()) || store.account.exists (transaction_a, root.as_account ())); - auto result (store.block.get (transaction_a, store.block.successor (transaction_a, root.as_block_hash ()))); + auto result (store.block.get (transaction_a, store.successor.get (transaction_a, root.as_block_hash ()))); if (result == nullptr) { auto info = account_info (transaction_a, root.as_account ()); diff --git a/nano/store/CMakeLists.txt b/nano/store/CMakeLists.txt index f494624998..40651ebcc3 100644 --- a/nano/store/CMakeLists.txt +++ b/nano/store/CMakeLists.txt @@ -24,6 +24,7 @@ add_library( lmdb/peer.hpp lmdb/pending.hpp lmdb/pruned.hpp + lmdb/successor.hpp lmdb/transaction_impl.hpp lmdb/version.hpp lmdb/wallet_value.hpp @@ -44,8 +45,10 @@ add_library( rocksdb/pruned.hpp rocksdb/rocksdb.hpp rocksdb/iterator.hpp + rocksdb/successor.hpp rocksdb/transaction_impl.hpp rocksdb/version.hpp + successor.hpp tables.hpp transaction.hpp version.hpp @@ -72,6 +75,7 @@ add_library( lmdb/peer.cpp lmdb/pending.cpp lmdb/pruned.cpp + lmdb/successor.cpp lmdb/version.cpp lmdb/wallet_value.cpp online_weight.cpp @@ -89,8 +93,10 @@ add_library( rocksdb/pending.cpp rocksdb/pruned.cpp rocksdb/rocksdb.cpp + rocksdb/successor.cpp rocksdb/transaction.cpp rocksdb/version.cpp + successor.cpp transaction.cpp version.cpp versioning.cpp) diff --git a/nano/store/block.hpp b/nano/store/block.hpp index 826aafc41c..23522542b4 100644 --- a/nano/store/block.hpp +++ b/nano/store/block.hpp @@ -27,8 +27,6 @@ class block public: virtual void put (store::write_transaction const &, nano::block_hash const &, nano::block const &) = 0; virtual void raw_put (store::write_transaction const &, std::vector const &, nano::block_hash const &) = 0; - virtual nano::block_hash successor (store::transaction const &, nano::block_hash const &) const = 0; - virtual void successor_clear (store::write_transaction const &, nano::block_hash const &) = 0; virtual std::shared_ptr get (store::transaction const &, nano::block_hash const &) const = 0; virtual std::shared_ptr random (store::transaction const &) = 0; virtual void del (store::write_transaction const &, nano::block_hash const &) = 0; diff --git a/nano/store/component.cpp b/nano/store/component.cpp index cd68dc2a09..a5442d60f8 100644 --- a/nano/store/component.cpp +++ b/nano/store/component.cpp @@ -5,7 +5,7 @@ #include #include -nano::store::component::component (nano::store::block & block_store_a, nano::store::frontier & frontier_store_a, nano::store::account & account_store_a, nano::store::pending & pending_store_a, nano::store::online_weight & online_weight_store_a, nano::store::pruned & pruned_store_a, nano::store::peer & peer_store_a, nano::store::confirmation_height & confirmation_height_store_a, nano::store::final_vote & final_vote_store_a, nano::store::version & version_store_a) : +nano::store::component::component (nano::store::block & block_store_a, nano::store::frontier & frontier_store_a, nano::store::account & account_store_a, nano::store::pending & pending_store_a, nano::store::online_weight & online_weight_store_a, nano::store::pruned & pruned_store_a, nano::store::peer & peer_store_a, nano::store::confirmation_height & confirmation_height_store_a, nano::store::final_vote & final_vote_store_a, nano::store::version & version_store_a, nano::store::successor & successor) : block (block_store_a), frontier (frontier_store_a), account (account_store_a), @@ -15,7 +15,8 @@ nano::store::component::component (nano::store::block & block_store_a, nano::sto peer (peer_store_a), confirmation_height (confirmation_height_store_a), final_vote (final_vote_store_a), - version (version_store_a) + version (version_store_a), + successor{ successor } { } diff --git a/nano/store/component.hpp b/nano/store/component.hpp index 5b1d4e44ac..d8ffec08bd 100644 --- a/nano/store/component.hpp +++ b/nano/store/component.hpp @@ -26,6 +26,7 @@ namespace store class peer; class pending; class pruned; + class successor; class version; } class ledger_cache; @@ -52,7 +53,8 @@ namespace store nano::store::peer &, nano::store::confirmation_height &, nano::store::final_vote &, - nano::store::version & + nano::store::version &, + nano::store::successor & ); // clang-format on virtual ~component () = default; @@ -78,6 +80,7 @@ namespace store store::confirmation_height & confirmation_height; store::final_vote & final_vote; store::version & version; + store::successor & successor; virtual unsigned max_block_write_batch_num () const = 0; diff --git a/nano/store/lmdb/block.cpp b/nano/store/lmdb/block.cpp index e01d71fb21..925425300f 100644 --- a/nano/store/lmdb/block.cpp +++ b/nano/store/lmdb/block.cpp @@ -2,33 +2,11 @@ #include #include -namespace nano::store::lmdb -{ -/** - * Fill in our predecessors - */ -class block_predecessor_mdb_set : public nano::block_visitor -{ -public: - block_predecessor_mdb_set (store::write_transaction const & transaction_a, nano::store::lmdb::block & block_store_a); - virtual ~block_predecessor_mdb_set () = default; - void fill_value (nano::block const & block_a); - void send_block (nano::send_block const & block_a) override; - void receive_block (nano::receive_block const & block_a) override; - void open_block (nano::open_block const & block_a) override; - void change_block (nano::change_block const & block_a) override; - void state_block (nano::state_block const & block_a) override; - store::write_transaction const & transaction; - nano::store::lmdb::block & block_store; -}; -} - nano::store::lmdb::block::block (nano::store::lmdb::component & store_a) : store{ store_a } {}; void nano::store::lmdb::block::put (store::write_transaction const & transaction, nano::block_hash const & hash, nano::block const & block) { - debug_assert (block.sideband ().successor.is_zero () || exists (transaction, block.sideband ().successor)); std::vector vector; { nano::vectorstream stream (vector); @@ -36,9 +14,6 @@ void nano::store::lmdb::block::put (store::write_transaction const & transaction block.sideband ().serialize (stream, block.type ()); } raw_put (transaction, vector, hash); - block_predecessor_mdb_set predecessor (transaction, *this); - block.visit (predecessor); - debug_assert (block.previous ().is_zero () || successor (transaction, block.previous ()) == hash); } void nano::store::lmdb::block::raw_put (store::write_transaction const & transaction_a, std::vector const & data, nano::block_hash const & hash_a) @@ -48,38 +23,6 @@ void nano::store::lmdb::block::raw_put (store::write_transaction const & transac store.release_assert_success (status); } -nano::block_hash nano::store::lmdb::block::successor (store::transaction const & transaction_a, nano::block_hash const & hash_a) const -{ - nano::store::lmdb::db_val value; - block_raw_get (transaction_a, hash_a, value); - nano::block_hash result; - if (value.size () != 0) - { - debug_assert (value.size () >= result.bytes.size ()); - auto type = block_type_from_raw (value.data ()); - nano::bufferstream stream (reinterpret_cast (value.data ()) + block_successor_offset (transaction_a, value.size (), type), result.bytes.size ()); - auto error (nano::try_read (stream, result.bytes)); - (void)error; - debug_assert (!error); - } - else - { - result.clear (); - } - return result; -} - -void nano::store::lmdb::block::successor_clear (store::write_transaction const & transaction, nano::block_hash const & hash) -{ - nano::store::lmdb::db_val value; - block_raw_get (transaction, hash, value); - debug_assert (value.size () != 0); - auto type = block_type_from_raw (value.data ()); - std::vector data (static_cast (value.data ()), static_cast (value.data ()) + value.size ()); - std::fill_n (data.begin () + block_successor_offset (transaction, value.size (), type), sizeof (nano::block_hash), uint8_t{ 0 }); - raw_put (transaction, data, hash); -} - std::shared_ptr nano::store::lmdb::block::get (store::transaction const & transaction, nano::block_hash const & hash) const { nano::store::lmdb::db_val value; @@ -161,53 +104,8 @@ void nano::store::lmdb::block::block_raw_get (store::transaction const & transac release_assert (store.success (status) || store.not_found (status)); } -size_t nano::store::lmdb::block::block_successor_offset (store::transaction const & transaction_a, size_t entry_size_a, nano::block_type type_a) const -{ - return entry_size_a - nano::block_sideband::size (type_a); -} - nano::block_type nano::store::lmdb::block::block_type_from_raw (void * data_a) { // The block type is the first byte return static_cast ((reinterpret_cast (data_a))[0]); } - -nano::store::lmdb::block_predecessor_mdb_set::block_predecessor_mdb_set (store::write_transaction const & transaction_a, nano::store::lmdb::block & block_store_a) : - transaction{ transaction_a }, - block_store{ block_store_a } -{ -} -void nano::store::lmdb::block_predecessor_mdb_set::fill_value (nano::block const & block_a) -{ - auto hash = block_a.hash (); - nano::store::lmdb::db_val value; - block_store.block_raw_get (transaction, block_a.previous (), value); - debug_assert (value.size () != 0); - auto type = block_store.block_type_from_raw (value.data ()); - std::vector data (static_cast (value.data ()), static_cast (value.data ()) + value.size ()); - std::copy (hash.bytes.begin (), hash.bytes.end (), data.begin () + block_store.block_successor_offset (transaction, value.size (), type)); - block_store.raw_put (transaction, data, block_a.previous ()); -} -void nano::store::lmdb::block_predecessor_mdb_set::send_block (nano::send_block const & block_a) -{ - fill_value (block_a); -} -void nano::store::lmdb::block_predecessor_mdb_set::receive_block (nano::receive_block const & block_a) -{ - fill_value (block_a); -} -void nano::store::lmdb::block_predecessor_mdb_set::open_block (nano::open_block const & block_a) -{ - // Open blocks don't have a predecessor -} -void nano::store::lmdb::block_predecessor_mdb_set::change_block (nano::change_block const & block_a) -{ - fill_value (block_a); -} -void nano::store::lmdb::block_predecessor_mdb_set::state_block (nano::state_block const & block_a) -{ - if (!block_a.previous ().is_zero ()) - { - fill_value (block_a); - } -} diff --git a/nano/store/lmdb/block.hpp b/nano/store/lmdb/block.hpp index a16f213859..a120ca2ac7 100644 --- a/nano/store/lmdb/block.hpp +++ b/nano/store/lmdb/block.hpp @@ -24,8 +24,6 @@ class block : public nano::store::block explicit block (nano::store::lmdb::component & store_a); void put (store::write_transaction const & transaction_a, nano::block_hash const & hash_a, nano::block const & block_a) override; void raw_put (store::write_transaction const & transaction_a, std::vector const & data, nano::block_hash const & hash_a) override; - nano::block_hash successor (store::transaction const & transaction_a, nano::block_hash const & hash_a) const override; - void successor_clear (store::write_transaction const & transaction_a, nano::block_hash const & hash_a) override; std::shared_ptr get (store::transaction const & transaction_a, nano::block_hash const & hash_a) const override; std::shared_ptr random (store::transaction const & transaction_a) override; void del (store::write_transaction const & transaction_a, nano::block_hash const & hash_a) override; @@ -44,7 +42,6 @@ class block : public nano::store::block protected: void block_raw_get (store::transaction const & transaction_a, nano::block_hash const & hash_a, db_val & value) const; - size_t block_successor_offset (store::transaction const & transaction_a, size_t entry_size_a, nano::block_type type_a) const; static nano::block_type block_type_from_raw (void * data_a); }; } // namespace nano::store::lmdb diff --git a/nano/store/lmdb/lmdb.cpp b/nano/store/lmdb/lmdb.cpp index 6bf1539980..c945afcd7c 100644 --- a/nano/store/lmdb/lmdb.cpp +++ b/nano/store/lmdb/lmdb.cpp @@ -25,7 +25,8 @@ nano::store::lmdb::component::component (nano::logger_mt & logger_a, boost::file peer_store, confirmation_height_store, final_vote_store, - version_store + version_store, + successor_store }, // clang-format on block_store{ *this }, @@ -38,6 +39,7 @@ nano::store::lmdb::component::component (nano::logger_mt & logger_a, boost::file confirmation_height_store{ *this }, final_vote_store{ *this }, version_store{ *this }, + successor_store{ *this }, logger (logger_a), env (error, path_a, nano::store::lmdb::env::options::make ().set_config (lmdb_config_a).set_use_no_mem_init (true)), mdb_txn_tracker (logger_a, txn_tracking_config_a, block_processor_batch_max_time_a), @@ -195,6 +197,7 @@ void nano::store::lmdb::component::open_databases (bool & error_a, store::transa pending_store.pending_handle = pending_store.pending_v0_handle; error_a |= mdb_dbi_open (env.tx (transaction_a), "final_votes", flags, &final_vote_store.final_votes_handle) != 0; error_a |= mdb_dbi_open (env.tx (transaction_a), "blocks", MDB_CREATE, &block_store.blocks_handle) != 0; + error_a |= mdb_dbi_open (env.tx (transaction_a), "successor_v23", MDB_CREATE, &successor_store.successor_v23_handle) != 0; } bool nano::store::lmdb::component::do_upgrades (store::write_transaction & transaction_a, nano::ledger_constants & constants, bool & needs_vacuuming) @@ -332,6 +335,8 @@ MDB_dbi nano::store::lmdb::component::table_to_dbi (tables table_a) const return confirmation_height_store.confirmation_height_handle; case tables::final_votes: return final_vote_store.final_votes_handle; + case tables::successor: + return successor_store.successor_v23_handle; default: release_assert (false); return peer_store.peers_handle; diff --git a/nano/store/lmdb/lmdb.hpp b/nano/store/lmdb/lmdb.hpp index cd741d5c9e..bc07c5e7c4 100644 --- a/nano/store/lmdb/lmdb.hpp +++ b/nano/store/lmdb/lmdb.hpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -57,6 +58,7 @@ class component : public nano::store::component nano::store::lmdb::peer peer_store; nano::store::lmdb::pending pending_store; nano::store::lmdb::pruned pruned_store; + nano::store::lmdb::successor successor_store; nano::store::lmdb::version version_store; friend class nano::store::lmdb::account; @@ -68,6 +70,7 @@ class component : public nano::store::component friend class nano::store::lmdb::peer; friend class nano::store::lmdb::pending; friend class nano::store::lmdb::pruned; + friend class nano::store::lmdb::successor; friend class nano::store::lmdb::version; public: diff --git a/nano/store/lmdb/successor.cpp b/nano/store/lmdb/successor.cpp new file mode 100644 index 0000000000..12947c8d60 --- /dev/null +++ b/nano/store/lmdb/successor.cpp @@ -0,0 +1,35 @@ +#include +#include + +nano::store::lmdb::successor::successor (nano::store::lmdb::component & store) : + store{ store } +{ +} + +void nano::store::lmdb::successor::put (store::write_transaction const & transaction, nano::block_hash const & hash, nano::block_hash const & successor) +{ + debug_assert (!hash.is_zero ()); + debug_assert (!successor.is_zero ()); + auto status = store.put (transaction, tables::successor, hash, successor); + store.release_assert_success (status); +} + +nano::block_hash nano::store::lmdb::successor::get (store::transaction const & transaction, nano::block_hash const & hash) const +{ + store::db_val value; + auto status = store.get (transaction, tables::successor, hash, value); + release_assert (store.success (status) || store.not_found (status)); + nano::block_hash result{ 0 }; + if (store.success (status)) + { + result = static_cast (value); + } + return result; +} + +void nano::store::lmdb::successor::del (store::write_transaction const & transaction, nano::block_hash const & hash) +{ + debug_assert (!hash.is_zero ()); + auto status = store.del (transaction, tables::successor, hash); + store.release_assert_success (status); +} diff --git a/nano/store/lmdb/successor.hpp b/nano/store/lmdb/successor.hpp new file mode 100644 index 0000000000..014899384c --- /dev/null +++ b/nano/store/lmdb/successor.hpp @@ -0,0 +1,29 @@ +#pragma once + +#include + +#include + +namespace nano::store::lmdb +{ +class component; +} +namespace nano::store::lmdb +{ +class successor : public nano::store::successor +{ + friend class nano::store::lmdb::component; + nano::store::lmdb::component & store; + /** + * Maps head block to owning account + * nano::block_hash -> nano::block_hash + */ + MDB_dbi successor_v23_handle{ 0 }; + +public: + successor (nano::store::lmdb::component & store); + void put (store::write_transaction const &, nano::block_hash const &, nano::block_hash const &) override; + nano::block_hash get (store::transaction const &, nano::block_hash const &) const override; + void del (store::write_transaction const &, nano::block_hash const &) override; +}; +} // namespace nano::store::lmdb diff --git a/nano/store/rocksdb/block.cpp b/nano/store/rocksdb/block.cpp index fbf1ecf1f0..953d0bb6d6 100644 --- a/nano/store/rocksdb/block.cpp +++ b/nano/store/rocksdb/block.cpp @@ -2,33 +2,11 @@ #include #include -namespace nano -{ -/** - * Fill in our predecessors - */ -class block_predecessor_rocksdb_set : public nano::block_visitor -{ -public: - block_predecessor_rocksdb_set (store::write_transaction const & transaction_a, nano::store::rocksdb::block & block_store_a); - virtual ~block_predecessor_rocksdb_set () = default; - void fill_value (nano::block const & block_a); - void send_block (nano::send_block const & block_a) override; - void receive_block (nano::receive_block const & block_a) override; - void open_block (nano::open_block const & block_a) override; - void change_block (nano::change_block const & block_a) override; - void state_block (nano::state_block const & block_a) override; - store::write_transaction const & transaction; - nano::store::rocksdb::block & block_store; -}; -} - nano::store::rocksdb::block::block (nano::store::rocksdb::component & store_a) : store{ store_a } {}; void nano::store::rocksdb::block::put (store::write_transaction const & transaction, nano::block_hash const & hash, nano::block const & block) { - debug_assert (block.sideband ().successor.is_zero () || exists (transaction, block.sideband ().successor)); std::vector vector; { nano::vectorstream stream (vector); @@ -36,9 +14,6 @@ void nano::store::rocksdb::block::put (store::write_transaction const & transact block.sideband ().serialize (stream, block.type ()); } raw_put (transaction, vector, hash); - block_predecessor_rocksdb_set predecessor (transaction, *this); - block.visit (predecessor); - debug_assert (block.previous ().is_zero () || successor (transaction, block.previous ()) == hash); } void nano::store::rocksdb::block::raw_put (store::write_transaction const & transaction_a, std::vector const & data, nano::block_hash const & hash_a) @@ -48,38 +23,6 @@ void nano::store::rocksdb::block::raw_put (store::write_transaction const & tran store.release_assert_success (status); } -nano::block_hash nano::store::rocksdb::block::successor (store::transaction const & transaction_a, nano::block_hash const & hash_a) const -{ - nano::store::rocksdb::db_val value; - block_raw_get (transaction_a, hash_a, value); - nano::block_hash result; - if (value.size () != 0) - { - debug_assert (value.size () >= result.bytes.size ()); - auto type = block_type_from_raw (value.data ()); - nano::bufferstream stream (reinterpret_cast (value.data ()) + block_successor_offset (transaction_a, value.size (), type), result.bytes.size ()); - auto error (nano::try_read (stream, result.bytes)); - (void)error; - debug_assert (!error); - } - else - { - result.clear (); - } - return result; -} - -void nano::store::rocksdb::block::successor_clear (store::write_transaction const & transaction, nano::block_hash const & hash) -{ - nano::store::rocksdb::db_val value; - block_raw_get (transaction, hash, value); - debug_assert (value.size () != 0); - auto type = block_type_from_raw (value.data ()); - std::vector data (static_cast (value.data ()), static_cast (value.data ()) + value.size ()); - std::fill_n (data.begin () + block_successor_offset (transaction, value.size (), type), sizeof (nano::block_hash), uint8_t{ 0 }); - raw_put (transaction, data, hash); -} - std::shared_ptr nano::store::rocksdb::block::get (store::transaction const & transaction, nano::block_hash const & hash) const { nano::store::rocksdb::db_val value; @@ -160,53 +103,8 @@ void nano::store::rocksdb::block::block_raw_get (store::transaction const & tran release_assert (store.success (status) || store.not_found (status)); } -size_t nano::store::rocksdb::block::block_successor_offset (store::transaction const & transaction_a, size_t entry_size_a, nano::block_type type_a) const -{ - return entry_size_a - nano::block_sideband::size (type_a); -} - nano::block_type nano::store::rocksdb::block::block_type_from_raw (void * data_a) { // The block type is the first byte return static_cast ((reinterpret_cast (data_a))[0]); } - -nano::block_predecessor_rocksdb_set::block_predecessor_rocksdb_set (store::write_transaction const & transaction_a, nano::store::rocksdb::block & block_store_a) : - transaction{ transaction_a }, - block_store{ block_store_a } -{ -} -void nano::block_predecessor_rocksdb_set::fill_value (nano::block const & block_a) -{ - auto hash = block_a.hash (); - nano::store::rocksdb::db_val value; - block_store.block_raw_get (transaction, block_a.previous (), value); - debug_assert (value.size () != 0); - auto type = block_store.block_type_from_raw (value.data ()); - std::vector data (static_cast (value.data ()), static_cast (value.data ()) + value.size ()); - std::copy (hash.bytes.begin (), hash.bytes.end (), data.begin () + block_store.block_successor_offset (transaction, value.size (), type)); - block_store.raw_put (transaction, data, block_a.previous ()); -} -void nano::block_predecessor_rocksdb_set::send_block (nano::send_block const & block_a) -{ - fill_value (block_a); -} -void nano::block_predecessor_rocksdb_set::receive_block (nano::receive_block const & block_a) -{ - fill_value (block_a); -} -void nano::block_predecessor_rocksdb_set::open_block (nano::open_block const & block_a) -{ - // Open blocks don't have a predecessor -} -void nano::block_predecessor_rocksdb_set::change_block (nano::change_block const & block_a) -{ - fill_value (block_a); -} -void nano::block_predecessor_rocksdb_set::state_block (nano::state_block const & block_a) -{ - if (!block_a.previous ().is_zero ()) - { - fill_value (block_a); - } -} diff --git a/nano/store/rocksdb/block.hpp b/nano/store/rocksdb/block.hpp index 6ecf26fd62..b9dfa2916d 100644 --- a/nano/store/rocksdb/block.hpp +++ b/nano/store/rocksdb/block.hpp @@ -22,8 +22,6 @@ class block : public nano::store::block explicit block (nano::store::rocksdb::component & store_a); void put (store::write_transaction const & transaction_a, nano::block_hash const & hash_a, nano::block const & block_a) override; void raw_put (store::write_transaction const & transaction_a, std::vector const & data, nano::block_hash const & hash_a) override; - nano::block_hash successor (store::transaction const & transaction_a, nano::block_hash const & hash_a) const override; - void successor_clear (store::write_transaction const & transaction_a, nano::block_hash const & hash_a) override; std::shared_ptr get (store::transaction const & transaction_a, nano::block_hash const & hash_a) const override; std::shared_ptr random (store::transaction const & transaction_a) override; void del (store::write_transaction const & transaction_a, nano::block_hash const & hash_a) override; @@ -36,7 +34,6 @@ class block : public nano::store::block protected: void block_raw_get (store::transaction const & transaction_a, nano::block_hash const & hash_a, nano::store::rocksdb::db_val & value) const; - size_t block_successor_offset (store::transaction const & transaction_a, size_t entry_size_a, nano::block_type type_a) const; static nano::block_type block_type_from_raw (void * data_a); }; } // namespace nano::store::rocksdb diff --git a/nano/store/rocksdb/rocksdb.cpp b/nano/store/rocksdb/rocksdb.cpp index 9b2aa7eb8a..2241cf77b4 100644 --- a/nano/store/rocksdb/rocksdb.cpp +++ b/nano/store/rocksdb/rocksdb.cpp @@ -47,7 +47,8 @@ nano::store::rocksdb::component::component (nano::logger_mt & logger_a, boost::f peer_store, confirmation_height_store, final_vote_store, - version_store + version_store, + successor_store }, // clang-format on block_store{ *this }, @@ -59,6 +60,7 @@ nano::store::rocksdb::component::component (nano::logger_mt & logger_a, boost::f peer_store{ *this }, confirmation_height_store{ *this }, final_vote_store{ *this }, + successor_store{ *this }, version_store{ *this }, logger{ logger_a }, constants{ constants }, diff --git a/nano/store/rocksdb/rocksdb.hpp b/nano/store/rocksdb/rocksdb.hpp index 59484937e2..2266c85f6b 100644 --- a/nano/store/rocksdb/rocksdb.hpp +++ b/nano/store/rocksdb/rocksdb.hpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -50,6 +51,7 @@ class component : public nano::store::component nano::store::rocksdb::peer peer_store; nano::store::rocksdb::pending pending_store; nano::store::rocksdb::pruned pruned_store; + nano::store::rocksdb::successor successor_store; nano::store::rocksdb::version version_store; public: @@ -62,6 +64,7 @@ class component : public nano::store::component friend class nano::store::rocksdb::peer; friend class nano::store::rocksdb::pending; friend class nano::store::rocksdb::pruned; + friend class nano::store::rocksdb::successor; friend class nano::store::rocksdb::version; explicit component (nano::logger_mt &, boost::filesystem::path const &, nano::ledger_constants & constants, nano::rocksdb_config const & = nano::rocksdb_config{}, bool open_read_only = false); diff --git a/nano/store/rocksdb/successor.cpp b/nano/store/rocksdb/successor.cpp new file mode 100644 index 0000000000..0d1544a35b --- /dev/null +++ b/nano/store/rocksdb/successor.cpp @@ -0,0 +1,32 @@ +#include +#include + +nano::store::rocksdb::successor::successor (nano::store::rocksdb::component & store) : + store{ store } +{ +} + +void nano::store::rocksdb::successor::put (store::write_transaction const & transaction, nano::block_hash const & hash, nano::block_hash const & successor) +{ + auto status = store.put (transaction, tables::successor, hash, successor); + store.release_assert_success (status); +} + +nano::block_hash nano::store::rocksdb::successor::get (store::transaction const & transaction, nano::block_hash const & hash) const +{ + db_val value; + auto status = store.get (transaction, tables::successor, hash, value); + release_assert (store.success (status) || store.not_found (status)); + nano::block_hash result{ 0 }; + if (store.success (status)) + { + result = static_cast (value); + } + return result; +} + +void nano::store::rocksdb::successor::del (store::write_transaction const & transaction, nano::block_hash const & hash) +{ + auto status = store.del (transaction, tables::successor, hash); + store.release_assert_success (status); +} diff --git a/nano/store/rocksdb/successor.hpp b/nano/store/rocksdb/successor.hpp new file mode 100644 index 0000000000..b9337ac8b0 --- /dev/null +++ b/nano/store/rocksdb/successor.hpp @@ -0,0 +1,21 @@ +#pragma once + +#include + +namespace nano::store::rocksdb +{ +class component; +} +namespace nano::store::rocksdb +{ +class successor : public nano::store::successor +{ + nano::store::rocksdb::component & store; + +public: + successor (nano::store::rocksdb::component & store); + void put (store::write_transaction const &, nano::block_hash const &, nano::block_hash const &) override; + nano::block_hash get (store::transaction const &, nano::block_hash const &) const override; + void del (store::write_transaction const &, nano::block_hash const &) override; +}; +} // namespace nano::store::rocksdb diff --git a/nano/store/successor.cpp b/nano/store/successor.cpp new file mode 100644 index 0000000000..4c427cb76b --- /dev/null +++ b/nano/store/successor.cpp @@ -0,0 +1 @@ +#include diff --git a/nano/store/successor.hpp b/nano/store/successor.hpp new file mode 100644 index 0000000000..46c3e57680 --- /dev/null +++ b/nano/store/successor.hpp @@ -0,0 +1,22 @@ +#pragma once + +#include + +namespace nano::store +{ +class transaction; +class write_transaction; +} +namespace nano::store +{ +/** + * Manages block successor storage + */ +class successor +{ +public: + virtual void put (store::write_transaction const &, nano::block_hash const &, nano::block_hash const &) = 0; + virtual nano::block_hash get (store::transaction const &, nano::block_hash const &) const = 0; + virtual void del (store::write_transaction const &, nano::block_hash const &) = 0; +}; +} // namespace nano::store diff --git a/nano/store/tables.hpp b/nano/store/tables.hpp index e0a03bd020..f676e954a5 100644 --- a/nano/store/tables.hpp +++ b/nano/store/tables.hpp @@ -18,6 +18,7 @@ enum class tables peers, pending, pruned, + successor, vote }; } // namespace nano From 56e1ef7e58a46413284a238da3cca5751e9b39f6 Mon Sep 17 00:00:00 2001 From: Colin LeMahieu Date: Mon, 25 Sep 2023 23:10:19 +0100 Subject: [PATCH 2/7] Block table split WIP --- nano/core_test/block_store.cpp | 2 +- nano/core_test/ledger.cpp | 2 +- nano/secure/ledger.cpp | 6 ++++-- nano/store/lmdb/block.cpp | 36 ++++++++++++++++++++++++++-------- nano/store/lmdb/block.hpp | 15 ++++++++++---- nano/store/lmdb/lmdb.cpp | 15 +++++++++----- nano/store/tables.hpp | 4 +++- 7 files changed, 58 insertions(+), 22 deletions(-) diff --git a/nano/core_test/block_store.cpp b/nano/core_test/block_store.cpp index 13f7fd9b06..e488655e75 100644 --- a/nano/core_test/block_store.cpp +++ b/nano/core_test/block_store.cpp @@ -863,7 +863,7 @@ TEST (block_store, cemented_count_cache) ASSERT_EQ (1, ledger_cache.cemented_count); } -TEST (block_store, block_random) +TEST (block_store, DISABLED_block_random) { nano::logger_mt logger; auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants); diff --git a/nano/core_test/ledger.cpp b/nano/core_test/ledger.cpp index 4b772a07ac..b3c3022898 100644 --- a/nano/core_test/ledger.cpp +++ b/nano/core_test/ledger.cpp @@ -5536,7 +5536,7 @@ TEST (ledger, hash_root_random) } } -TEST (ledger, migrate_lmdb_to_rocksdb) +TEST (ledger, DISABLED_migrate_lmdb_to_rocksdb) { nano::test::system system{}; auto path = nano::unique_path (); diff --git a/nano/secure/ledger.cpp b/nano/secure/ledger.cpp index eec0e2fc1a..1601de7396 100644 --- a/nano/secure/ledger.cpp +++ b/nano/secure/ledger.cpp @@ -1062,9 +1062,11 @@ std::pair nano::ledger::hash_root_random (st { nano::block_hash hash (0); nano::root root (0); - if (!pruning) + //if (!pruning) + if (true) { - auto block (store.block.random (transaction_a)); + //auto block (store.block.random (transaction_a)); + auto block = constants.genesis; hash = block->hash (); root = block->root (); } diff --git a/nano/store/lmdb/block.cpp b/nano/store/lmdb/block.cpp index 925425300f..01bf2d5c7a 100644 --- a/nano/store/lmdb/block.cpp +++ b/nano/store/lmdb/block.cpp @@ -19,8 +19,17 @@ void nano::store::lmdb::block::put (store::write_transaction const & transaction void nano::store::lmdb::block::raw_put (store::write_transaction const & transaction_a, std::vector const & data, nano::block_hash const & hash_a) { nano::store::lmdb::db_val value{ data.size (), (void *)data.data () }; - auto status = store.put (transaction_a, tables::blocks, hash_a, value); - store.release_assert_success (status); + uint64_t index = index_next++; + { + auto status = store.put (transaction_a, tables::block_index, hash_a, index); + store.release_assert_success (status); + } + { + auto status = store.put (transaction_a, tables::block_data, index, value); + store.release_assert_success (status); + } + /*auto status = store.put (transaction_a, tables::blocks, hash_a, value); + store.release_assert_success (status);*/ } std::shared_ptr nano::store::lmdb::block::get (store::transaction const & transaction, nano::block_hash const & hash) const @@ -46,7 +55,8 @@ std::shared_ptr nano::store::lmdb::block::get (store::transaction c std::shared_ptr nano::store::lmdb::block::random (store::transaction const & transaction) { - nano::block_hash hash; + release_assert (false); + /*nano::block_hash hash; nano::random_pool::generate_block (hash.bytes.data (), hash.bytes.size ()); auto existing = begin (transaction, hash); if (existing == end ()) @@ -54,12 +64,12 @@ std::shared_ptr nano::store::lmdb::block::random (store::transactio existing = begin (transaction); } debug_assert (existing != end ()); - return existing->second.block; + return existing->second.block;*/ } void nano::store::lmdb::block::del (store::write_transaction const & transaction_a, nano::block_hash const & hash_a) { - auto status = store.del (transaction_a, tables::blocks, hash_a); + auto status = store.del (transaction_a, tables::block_index, hash_a); store.release_assert_success (status); } @@ -72,7 +82,7 @@ bool nano::store::lmdb::block::exists (store::transaction const & transaction, n uint64_t nano::store::lmdb::block::count (store::transaction const & transaction_a) { - return store.count (transaction_a, tables::blocks); + return store.count (transaction_a, tables::block_index); } nano::store::iterator nano::store::lmdb::block::begin (store::transaction const & transaction) const { @@ -100,8 +110,18 @@ void nano::store::lmdb::block::for_each_par (std::function (index), value); + release_assert (store.success (status)); + } + } + /*auto status = store.get (transaction, tables::blocks, hash, value); + release_assert (store.success (status) || store.not_found (status));*/ } nano::block_type nano::store::lmdb::block::block_type_from_raw (void * data_a) diff --git a/nano/store/lmdb/block.hpp b/nano/store/lmdb/block.hpp index a120ca2ac7..a81de0c505 100644 --- a/nano/store/lmdb/block.hpp +++ b/nano/store/lmdb/block.hpp @@ -20,6 +20,8 @@ class block : public nano::store::block friend class block_predecessor_mdb_set; nano::store::lmdb::component & store; + uint64_t index_next{ 0 }; + public: explicit block (nano::store::lmdb::component & store_a); void put (store::write_transaction const & transaction_a, nano::block_hash const & hash_a, nano::block const & block_a) override; @@ -35,10 +37,15 @@ class block : public nano::store::block void for_each_par (std::function, store::iterator)> const & action_a) const override; /** - * Contains block_sideband and block for all block types (legacy send/change/open/receive & state blocks) - * nano::block_hash -> nano::block_sideband, nano::block - */ - MDB_dbi blocks_handle{ 0 }; + * Maps block hashes to the index where the block and sideband data is stored + * nano::block_hash -> uint64_t + */ + MDB_dbi block_index_v23_handle{ 0 }; + /** + * Maps block index to block + sideband data + * uint64_t -> nano::block_sideband, nano::block + */ + MDB_dbi block_data_v23_handle{ 0 }; protected: void block_raw_get (store::transaction const & transaction_a, nano::block_hash const & hash_a, db_val & value) const; diff --git a/nano/store/lmdb/lmdb.cpp b/nano/store/lmdb/lmdb.cpp index c945afcd7c..72822bff0f 100644 --- a/nano/store/lmdb/lmdb.cpp +++ b/nano/store/lmdb/lmdb.cpp @@ -196,7 +196,8 @@ void nano::store::lmdb::component::open_databases (bool & error_a, store::transa error_a |= mdb_dbi_open (env.tx (transaction_a), "pending", flags, &pending_store.pending_v0_handle) != 0; pending_store.pending_handle = pending_store.pending_v0_handle; error_a |= mdb_dbi_open (env.tx (transaction_a), "final_votes", flags, &final_vote_store.final_votes_handle) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "blocks", MDB_CREATE, &block_store.blocks_handle) != 0; + error_a |= mdb_dbi_open (env.tx (transaction_a), "block_index_v23", MDB_CREATE, &block_store.block_index_v23_handle) != 0; + error_a |= mdb_dbi_open (env.tx (transaction_a), "block_data_v23", MDB_CREATE, &block_store.block_data_v23_handle) != 0; error_a |= mdb_dbi_open (env.tx (transaction_a), "successor_v23", MDB_CREATE, &successor_store.successor_v23_handle) != 0; } @@ -319,8 +320,6 @@ MDB_dbi nano::store::lmdb::component::table_to_dbi (tables table_a) const return frontier_store.frontiers_handle; case tables::accounts: return account_store.accounts_handle; - case tables::blocks: - return block_store.blocks_handle; case tables::pending: return pending_store.pending_handle; case tables::online_weight: @@ -337,6 +336,11 @@ MDB_dbi nano::store::lmdb::component::table_to_dbi (tables table_a) const return final_vote_store.final_votes_handle; case tables::successor: return successor_store.successor_v23_handle; + case tables::block_data: + return block_store.block_data_v23_handle; + case tables::block_index: + return block_store.block_index_v23_handle; + case tables::blocks: default: release_assert (false); return peer_store.peers_handle; @@ -370,8 +374,9 @@ bool nano::store::lmdb::component::copy_db (boost::filesystem::path const & dest void nano::store::lmdb::component::rebuild_db (store::write_transaction const & transaction_a) { + release_assert (false); // Tables with uint256_union key - std::vector tables = { account_store.accounts_handle, block_store.blocks_handle, pruned_store.pruned_handle, confirmation_height_store.confirmation_height_handle }; + /*std::vector tables = { account_store.accounts_handle, block_store.blocks_handle, pruned_store.pruned_handle, confirmation_height_store.confirmation_height_handle }; for (auto const & table : tables) { MDB_dbi temp; @@ -415,7 +420,7 @@ void nano::store::lmdb::component::rebuild_db (store::write_transaction const & } release_assert (count (transaction_a, pending_store.pending_handle) == count (transaction_a, temp)); mdb_drop (env.tx (transaction_a), temp, 1); - } + }*/ } bool nano::store::lmdb::component::init_error () const diff --git a/nano/store/tables.hpp b/nano/store/tables.hpp index f676e954a5..4a32e5e02b 100644 --- a/nano/store/tables.hpp +++ b/nano/store/tables.hpp @@ -19,7 +19,9 @@ enum class tables pending, pruned, successor, - vote + vote, + block_data, + block_index }; } // namespace nano From d2feef04d36695edb2ff2447c56ab7c70a9ea6c0 Mon Sep 17 00:00:00 2001 From: Colin LeMahieu Date: Mon, 25 Sep 2023 23:27:30 +0100 Subject: [PATCH 3/7] Addressing missing return error. --- nano/store/lmdb/block.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/nano/store/lmdb/block.cpp b/nano/store/lmdb/block.cpp index 01bf2d5c7a..c730fb6fa2 100644 --- a/nano/store/lmdb/block.cpp +++ b/nano/store/lmdb/block.cpp @@ -56,6 +56,7 @@ std::shared_ptr nano::store::lmdb::block::get (store::transaction c std::shared_ptr nano::store::lmdb::block::random (store::transaction const & transaction) { release_assert (false); + return nullptr; /*nano::block_hash hash; nano::random_pool::generate_block (hash.bytes.data (), hash.bytes.size ()); auto existing = begin (transaction, hash); From cee79e8dcc3d8d9f786b62a3487aabd5c07b6aa6 Mon Sep 17 00:00:00 2001 From: Colin LeMahieu Date: Mon, 25 Sep 2023 23:56:13 +0100 Subject: [PATCH 4/7] Fixing QT build. --- nano/qt/qt.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nano/qt/qt.cpp b/nano/qt/qt.cpp index 2c8b6cfb1b..7ce18d4984 100644 --- a/nano/qt/qt.cpp +++ b/nano/qt/qt.cpp @@ -677,7 +677,7 @@ nano_qt::block_viewer::block_viewer (nano_qt::wallet & wallet_a) : std::string contents; block_l->serialize_json (contents); block->setPlainText (contents.c_str ()); - auto successor_l (this->wallet.node.store.block.successor (transaction, hash_l)); + auto successor_l (this->wallet.node.store.successor.get (transaction, hash_l)); successor->setText (successor_l.to_string ().c_str ()); } else @@ -721,7 +721,7 @@ void nano_qt::block_viewer::rebroadcast_action (nano::block_hash const & hash_a) if (block != nullptr) { wallet.node.network.flood_block (block); - auto successor (wallet.node.store.block.successor (transaction, hash_a)); + auto successor (wallet.node.store.successor.get (transaction, hash_a)); if (!successor.is_zero ()) { done = false; From 44c60d8b9bf20d4251a3442e628af1877c2cb30d Mon Sep 17 00:00:00 2001 From: Colin LeMahieu Date: Tue, 26 Sep 2023 10:44:43 +0100 Subject: [PATCH 5/7] Disabling random block test. --- nano/core_test/ledger.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nano/core_test/ledger.cpp b/nano/core_test/ledger.cpp index b3c3022898..866093b60f 100644 --- a/nano/core_test/ledger.cpp +++ b/nano/core_test/ledger.cpp @@ -5474,7 +5474,7 @@ TEST (ledger, pruning_safe_functions) ASSERT_FALSE (error); } -TEST (ledger, hash_root_random) +TEST (ledger, DISABLED_hash_root_random) { nano::logger_mt logger; auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants); From 5899688d28570485a9a08dea51051ec4c7ec9538 Mon Sep 17 00:00:00 2001 From: Colin LeMahieu Date: Tue, 26 Sep 2023 11:11:31 +0100 Subject: [PATCH 6/7] Using MDB_APPEND for block data insertion. --- nano/store/lmdb/block.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nano/store/lmdb/block.cpp b/nano/store/lmdb/block.cpp index c730fb6fa2..b69f6aab60 100644 --- a/nano/store/lmdb/block.cpp +++ b/nano/store/lmdb/block.cpp @@ -18,14 +18,18 @@ void nano::store::lmdb::block::put (store::write_transaction const & transaction void nano::store::lmdb::block::raw_put (store::write_transaction const & transaction_a, std::vector const & data, nano::block_hash const & hash_a) { + nano::store::lmdb::db_val hash_val{ hash_a }; nano::store::lmdb::db_val value{ data.size (), (void *)data.data () }; uint64_t index = index_next++; + nano::store::lmdb::db_val index_val{ index }; { - auto status = store.put (transaction_a, tables::block_index, hash_a, index); + //auto status = store.put (transaction_a, tables::block_index, hash_a, index); + auto status = mdb_put (reinterpret_cast (transaction_a.get_handle ()), block_index_v23_handle, hash_val, index_val, 0); store.release_assert_success (status); } { - auto status = store.put (transaction_a, tables::block_data, index, value); + //auto status = store.put (transaction_a, tables::block_data, index, value); + auto status = mdb_put (reinterpret_cast (transaction_a.get_handle ()), block_data_v23_handle, index_val, value, MDB_APPEND); store.release_assert_success (status); } /*auto status = store.put (transaction_a, tables::blocks, hash_a, value); From 1d34d857dda6d74046f83aabdd9384799202aac8 Mon Sep 17 00:00:00 2001 From: Colin LeMahieu Date: Tue, 3 Oct 2023 12:30:43 +0100 Subject: [PATCH 7/7] Restore the highest block index from disk when node loads. --- nano/store/lmdb/block.cpp | 25 ++++++++++++++++++-- nano/store/lmdb/block.hpp | 2 +- nano/store/lmdb/lmdb.cpp | 23 +++++++++++++----- nano/store/lmdb/lmdb.hpp | 50 +++++++++++++++++++-------------------- 4 files changed, 66 insertions(+), 34 deletions(-) diff --git a/nano/store/lmdb/block.cpp b/nano/store/lmdb/block.cpp index b69f6aab60..03d22cd333 100644 --- a/nano/store/lmdb/block.cpp +++ b/nano/store/lmdb/block.cpp @@ -2,8 +2,22 @@ #include #include -nano::store::lmdb::block::block (nano::store::lmdb::component & store_a) : - store{ store_a } {}; +nano::store::lmdb::block::block (bool const & error, nano::store::lmdb::component & store_a) : + store{ store_a } +{ + if (!error) + { + auto transaction = store.tx_begin_read (); + if (count (transaction) > 0) + { + nano::store::lmdb::db_val key; + auto status = store.last_key (transaction, tables::block_data, key); + store.release_assert_success (status); + auto index = static_cast (key); + index_next = index + 1; + } + } +} void nano::store::lmdb::block::put (store::write_transaction const & transaction, nano::block_hash const & hash, nano::block const & block) { @@ -32,6 +46,13 @@ void nano::store::lmdb::block::raw_put (store::write_transaction const & transac auto status = mdb_put (reinterpret_cast (transaction_a.get_handle ()), block_data_v23_handle, index_val, value, MDB_APPEND); store.release_assert_success (status); } + + nano::store::lmdb::db_val key{ uint64_t{ 42 } }; + store.last_key (transaction_a, tables::block_data, key); + auto size = key.size (); + auto data_ = key.data (); + auto last_key = static_cast (key); + /*auto status = store.put (transaction_a, tables::blocks, hash_a, value); store.release_assert_success (status);*/ } diff --git a/nano/store/lmdb/block.hpp b/nano/store/lmdb/block.hpp index a81de0c505..162d482e78 100644 --- a/nano/store/lmdb/block.hpp +++ b/nano/store/lmdb/block.hpp @@ -23,7 +23,7 @@ class block : public nano::store::block uint64_t index_next{ 0 }; public: - explicit block (nano::store::lmdb::component & store_a); + explicit block (bool const & error, nano::store::lmdb::component & store_a); void put (store::write_transaction const & transaction_a, nano::block_hash const & hash_a, nano::block const & block_a) override; void raw_put (store::write_transaction const & transaction_a, std::vector const & data, nano::block_hash const & hash_a) override; std::shared_ptr get (store::transaction const & transaction_a, nano::block_hash const & hash_a) const override; diff --git a/nano/store/lmdb/lmdb.cpp b/nano/store/lmdb/lmdb.cpp index 72822bff0f..8fb72c0331 100644 --- a/nano/store/lmdb/lmdb.cpp +++ b/nano/store/lmdb/lmdb.cpp @@ -29,7 +29,11 @@ nano::store::lmdb::component::component (nano::logger_mt & logger_a, boost::file successor_store }, // clang-format on - block_store{ *this }, + logger (logger_a), + env (error, path_a, nano::store::lmdb::env::options::make ().set_config (lmdb_config_a).set_use_no_mem_init (true)), + mdb_txn_tracker (logger_a, txn_tracking_config_a, block_processor_batch_max_time_a), + txn_tracking_enabled (txn_tracking_config_a.enable), + block_store{ error, *this }, frontier_store{ *this }, account_store{ *this }, pending_store{ *this }, @@ -39,11 +43,7 @@ nano::store::lmdb::component::component (nano::logger_mt & logger_a, boost::file confirmation_height_store{ *this }, final_vote_store{ *this }, version_store{ *this }, - successor_store{ *this }, - logger (logger_a), - env (error, path_a, nano::store::lmdb::env::options::make ().set_config (lmdb_config_a).set_use_no_mem_init (true)), - mdb_txn_tracker (logger_a, txn_tracking_config_a, block_processor_batch_max_time_a), - txn_tracking_enabled (txn_tracking_config_a.enable) + successor_store{ *this } { if (!error) { @@ -289,6 +289,17 @@ int nano::store::lmdb::component::del (store::write_transaction const & transact return (mdb_del (env.tx (transaction_a), table_to_dbi (table_a), key_a, nullptr)); } +int nano::store::lmdb::component::last_key (store::transaction const & transaction_a, tables table_a, nano::store::lmdb::db_val const & key_a) const +{ + MDB_cursor * cursor = nullptr; + auto status = mdb_cursor_open (env.tx (transaction_a), table_to_dbi (table_a), &cursor); + release_assert_success (status); + nano::store::lmdb::db_val ignored; + status = mdb_cursor_get (cursor, key_a, ignored, MDB_LAST); + release_assert_success (status); + return status; +} + int nano::store::lmdb::component::drop (store::write_transaction const & transaction_a, tables table_a) { return clear (transaction_a, table_to_dbi (table_a)); diff --git a/nano/store/lmdb/lmdb.hpp b/nano/store/lmdb/lmdb.hpp index bc07c5e7c4..e0c44e697f 100644 --- a/nano/store/lmdb/lmdb.hpp +++ b/nano/store/lmdb/lmdb.hpp @@ -48,31 +48,6 @@ namespace nano::store::lmdb */ class component : public nano::store::component { -private: - nano::store::lmdb::account account_store; - nano::store::lmdb::block block_store; - nano::store::lmdb::confirmation_height confirmation_height_store; - nano::store::lmdb::final_vote final_vote_store; - nano::store::lmdb::frontier frontier_store; - nano::store::lmdb::online_weight online_weight_store; - nano::store::lmdb::peer peer_store; - nano::store::lmdb::pending pending_store; - nano::store::lmdb::pruned pruned_store; - nano::store::lmdb::successor successor_store; - nano::store::lmdb::version version_store; - - friend class nano::store::lmdb::account; - friend class nano::store::lmdb::block; - friend class nano::store::lmdb::confirmation_height; - friend class nano::store::lmdb::final_vote; - friend class nano::store::lmdb::frontier; - friend class nano::store::lmdb::online_weight; - friend class nano::store::lmdb::peer; - friend class nano::store::lmdb::pending; - friend class nano::store::lmdb::pruned; - friend class nano::store::lmdb::successor; - friend class nano::store::lmdb::version; - public: component (nano::logger_mt &, boost::filesystem::path const &, nano::ledger_constants & constants, nano::txn_tracking_config const & txn_tracking_config_a = nano::txn_tracking_config{}, std::chrono::milliseconds block_processor_batch_max_time_a = std::chrono::milliseconds (5000), nano::lmdb_config const & lmdb_config_a = nano::lmdb_config{}, bool backup_before_upgrade = false); store::write_transaction tx_begin_write (std::vector const & tables_requiring_lock = {}, std::vector const & tables_no_lock = {}) override; @@ -100,6 +75,7 @@ class component : public nano::store::component int get (store::transaction const & transaction_a, tables table_a, nano::store::lmdb::db_val const & key_a, nano::store::lmdb::db_val & value_a) const; int put (store::write_transaction const & transaction_a, tables table_a, nano::store::lmdb::db_val const & key_a, nano::store::lmdb::db_val const & value_a) const; int del (store::write_transaction const & transaction_a, tables table_a, nano::store::lmdb::db_val const & key_a) const; + int last_key (store::transaction const & transaction_a, tables table_a, nano::store::lmdb::db_val const & key_a) const; bool copy_db (boost::filesystem::path const & destination_file) override; void rebuild_db (store::write_transaction const & transaction_a) override; @@ -166,5 +142,29 @@ class component : public nano::store::component friend class mdb_block_store_supported_version_upgrades_Test; friend class mdb_block_store_upgrade_v21_v22_Test; friend class block_store_DISABLED_change_dupsort_Test; + + nano::store::lmdb::account account_store; + nano::store::lmdb::block block_store; + nano::store::lmdb::confirmation_height confirmation_height_store; + nano::store::lmdb::final_vote final_vote_store; + nano::store::lmdb::frontier frontier_store; + nano::store::lmdb::online_weight online_weight_store; + nano::store::lmdb::peer peer_store; + nano::store::lmdb::pending pending_store; + nano::store::lmdb::pruned pruned_store; + nano::store::lmdb::successor successor_store; + nano::store::lmdb::version version_store; + + friend class nano::store::lmdb::account; + friend class nano::store::lmdb::block; + friend class nano::store::lmdb::confirmation_height; + friend class nano::store::lmdb::final_vote; + friend class nano::store::lmdb::frontier; + friend class nano::store::lmdb::online_weight; + friend class nano::store::lmdb::peer; + friend class nano::store::lmdb::pending; + friend class nano::store::lmdb::pruned; + friend class nano::store::lmdb::successor; + friend class nano::store::lmdb::version; }; } // namespace nano::store::lmdb