diff --git a/.gitmodules b/.gitmodules index 36f0bcd46b..d37697838b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,8 +3,7 @@ url = https://github.com/miniupnp/miniupnp.git [submodule "lmdb"] path = submodules/lmdb - url = https://github.com/nanocurrency/lmdb.git - branch = lmdb_0_9_23 + url = https://github.com/LMDB/lmdb.git [submodule "cryptopp"] path = submodules/cryptopp url = https://github.com/weidai11/cryptopp.git diff --git a/CMakeLists.txt b/CMakeLists.txt index f3e9c3c316..c8cd2ec424 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.14) if(CMAKE_VERSION VERSION_GREATER 3.12 OR CMAKE_VERSION VERSION_EQUAL 3.12) # find_package uses _ROOT variables @@ -547,7 +547,8 @@ endif() if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") add_definitions(-DCRYPTOPP_DISABLE_MIXED_ASM -DCRYPTOPP_DISABLE_ASM) message( - "CryptoPP with disabled ASM for ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}" + STATUS + "CryptoPP with disabled ASM for ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}" ) endif() add_definitions(-DCRYPTOPP_DISABLE_SHANI) @@ -650,6 +651,9 @@ target_compile_definitions(blake2 PRIVATE -D__SSE2__) add_subdirectory(nano/crypto_lib) add_subdirectory(nano/secure) +add_subdirectory(nano/store) +target_include_directories(nano_store PUBLIC ${BOOST_LIBRARY_INCLUDES}) +target_include_directories(nano_store PUBLIC submodules) add_subdirectory(nano/lib) add_subdirectory(nano/node) add_subdirectory(nano/nano_node) @@ -774,7 +778,7 @@ if(NANO_GUI OR RAIBLOCKS_GUI) nano_wallet ${PLATFORM_GUI_TARGET_PROPERTIES} ${PLATFORM_WALLET_SOURCE} nano/nano_wallet/entry.cpp nano/nano_wallet/icon.hpp ${RES}) - target_link_libraries(nano_wallet rpc node qt) + target_link_libraries(nano_wallet Boost::process rpc node qt) if(WIN32) target_link_libraries(nano_wallet Qt5::WinExtras) diff --git a/nano/core_test/backlog.cpp b/nano/core_test/backlog.cpp index 0ad651d864..facbf0b538 100644 --- a/nano/core_test/backlog.cpp +++ b/nano/core_test/backlog.cpp @@ -20,7 +20,7 @@ TEST (backlog, population) nano::test::system system{}; auto & node = *system.add_node (); - node.backlog.activate_callback.add ([&] (nano::transaction const & transaction, nano::account const & account, nano::account_info const & account_info, nano::confirmation_height_info const & conf_info) { + node.backlog.activate_callback.add ([&] (nano::store::transaction const & transaction, nano::account const & account, nano::account_info const & account_info, nano::confirmation_height_info const & conf_info) { nano::lock_guard lock{ mutex }; activated.insert (account); diff --git a/nano/core_test/block_store.cpp b/nano/core_test/block_store.cpp index 08f0bd9b7c..573f412eb4 100644 --- a/nano/core_test/block_store.cpp +++ b/nano/core_test/block_store.cpp @@ -5,11 +5,14 @@ #include #include #include -#include -#include +#include #include #include -#include +#include +#include +#include +#include +#include #include #include @@ -24,18 +27,6 @@ using namespace std::chrono_literals; -namespace nano -{ -namespace lmdb -{ - void modify_account_info_to_v14 (nano::lmdb::store & store, nano::transaction const & transaction_a, nano::account const & account_a, uint64_t confirmation_height, nano::block_hash const & rep_block); - void modify_confirmation_height_to_v15 (nano::lmdb::store & store, nano::transaction const & transaction, nano::account const & account, uint64_t confirmation_height); - void write_sideband_v14 (nano::lmdb::store & store_a, nano::transaction & transaction_a, nano::block const & block_a, MDB_dbi db_a); - void write_sideband_v15 (nano::lmdb::store & store_a, nano::transaction & transaction_a, nano::block const & block_a); - void write_block_w_sideband_v18 (nano::lmdb::store & store_a, MDB_dbi database, nano::write_transaction & transaction_a, nano::block const & block_a); -} -} - TEST (block_store, construction) { nano::logger_mt logger; @@ -627,60 +618,50 @@ TEST (block_store, latest_find) ASSERT_EQ (second, find3); } -namespace nano +namespace nano::store::lmdb { -namespace lmdb +TEST (mdb_block_store, supported_version_upgrades) { - TEST (mdb_block_store, supported_version_upgrades) + if (nano::rocksdb_config::using_rocksdb_in_tests ()) { - if (nano::rocksdb_config::using_rocksdb_in_tests ()) - { - // Don't test this in rocksdb mode - GTEST_SKIP (); - } - // Check that upgrading from an unsupported version is not supported - auto path (nano::unique_path () / "data.ldb"); - nano::logger_mt logger; - { - nano::lmdb::store store (logger, path, nano::dev::constants); - nano::stats stats; - nano::ledger ledger (store, stats, nano::dev::constants); - auto transaction (store.tx_begin_write ()); - store.initialize (transaction, ledger.cache, nano::dev::constants); - // Lower the database to the max version unsupported for upgrades - store.version.put (transaction, store.version_minimum - 1); - } + // Don't test this in rocksdb mode + GTEST_SKIP (); + } + // Check that upgrading from an unsupported version is not supported + auto path (nano::unique_path () / "data.ldb"); + nano::logger_mt logger; + { + nano::store::lmdb::component store (logger, path, nano::dev::constants); + nano::stats stats; + nano::ledger ledger (store, stats, nano::dev::constants); + auto transaction (store.tx_begin_write ()); + store.initialize (transaction, ledger.cache, nano::dev::constants); + // Lower the database to the max version unsupported for upgrades + store.version.put (transaction, store.version_minimum - 1); + } - // Upgrade should fail - { - nano::lmdb::store store (logger, path, nano::dev::constants); - ASSERT_TRUE (store.init_error ()); - } + // Upgrade should fail + { + nano::store::lmdb::component store (logger, path, nano::dev::constants); + ASSERT_TRUE (store.init_error ()); + } - auto path1 (nano::unique_path () / "data.ldb"); - // Now try with the minimum version - { - nano::lmdb::store store (logger, path1, nano::dev::constants); - nano::stats stats; - nano::ledger ledger (store, stats, nano::dev::constants); - auto transaction (store.tx_begin_write ()); - store.initialize (transaction, ledger.cache, nano::dev::constants); - // Lower the database version to the minimum version supported for upgrade. - store.version.put (transaction, store.version_minimum); - store.confirmation_height.del (transaction, nano::dev::genesis->account ()); - ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "accounts_v1", MDB_CREATE, - &store.account_store.accounts_v1_handle)); - ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "open", MDB_CREATE, &store.block_store.open_blocks_handle)); - modify_account_info_to_v14 (store, transaction, nano::dev::genesis->account (), 1, - nano::dev::genesis->hash ()); - write_block_w_sideband_v18 (store, store.block_store.open_blocks_handle, transaction, *nano::dev::genesis); - } + auto path1 (nano::unique_path () / "data.ldb"); + // Now try with the minimum version + { + nano::store::lmdb::component store (logger, path1, nano::dev::constants); + nano::stats stats; + nano::ledger ledger (store, stats, nano::dev::constants); + auto transaction (store.tx_begin_write ()); + store.initialize (transaction, ledger.cache, nano::dev::constants); + // Lower the database version to the minimum version supported for upgrade. + store.version.put (transaction, store.version_minimum); + } - // Upgrade should work - { - nano::lmdb::store store (logger, path1, nano::dev::constants); - ASSERT_FALSE (store.init_error ()); - } + // Upgrade should work + { + nano::store::lmdb::component store (logger, path1, nano::dev::constants); + ASSERT_FALSE (store.init_error ()); } } } @@ -693,7 +674,7 @@ TEST (mdb_block_store, bad_path) GTEST_SKIP (); } nano::logger_mt logger; - nano::lmdb::store store (logger, boost::filesystem::path ("///"), nano::dev::constants); + nano::store::lmdb::component store (logger, boost::filesystem::path ("///"), nano::dev::constants); ASSERT_TRUE (store.init_error ()); } @@ -1012,7 +993,7 @@ TEST (mdb_block_store, sideband_height) nano::keypair key1; nano::keypair key2; nano::keypair key3; - nano::lmdb::store store (logger, nano::unique_path () / "data.ldb", nano::dev::constants); + nano::store::lmdb::component store (logger, nano::unique_path () / "data.ldb", nano::dev::constants); ASSERT_FALSE (store.init_error ()); nano::stats stats; nano::ledger ledger (store, stats, nano::dev::constants); @@ -1100,7 +1081,7 @@ TEST (mdb_block_store, sideband_height) .work (*pool.generate (state_open->hash ())) .build (); ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, *epoch).code); - ASSERT_EQ (nano::epoch::epoch_1, store.block.version (transaction, epoch->hash ())); + ASSERT_EQ (nano::epoch::epoch_1, ledger.version (*epoch)); auto epoch_open = builder .state () .account (key2.pub) @@ -1112,7 +1093,7 @@ TEST (mdb_block_store, sideband_height) .work (*pool.generate (key2.pub)) .build (); ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, *epoch_open).code); - ASSERT_EQ (nano::epoch::epoch_1, store.block.version (transaction, epoch_open->hash ())); + ASSERT_EQ (nano::epoch::epoch_1, ledger.version (*epoch_open)); auto state_receive = builder .state () .account (key2.pub) @@ -1370,896 +1351,89 @@ TEST (block_store, pruned_blocks) ASSERT_EQ (store->pruned.count (store->tx_begin_read ()), 0); } -namespace nano +namespace nano::store::lmdb { -namespace lmdb +TEST (mdb_block_store, upgrade_v21_v22) { - TEST (mdb_block_store, upgrade_v14_v15) + if (nano::rocksdb_config::using_rocksdb_in_tests ()) { - if (nano::rocksdb_config::using_rocksdb_in_tests ()) - { - // Don't test this in rocksdb mode - GTEST_SKIP (); - } - // Extract confirmation height to a separate database - auto path (nano::unique_path () / "data.ldb"); - nano::block_builder builder; - nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits::max () }; - auto send = builder - .send () - .previous (nano::dev::genesis->hash ()) - .destination (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount - nano::Gxrb_ratio) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*pool.generate (nano::dev::genesis->hash ())) - .build (); - auto epoch = builder - .state () - .account (nano::dev::genesis_key.pub) - .previous (send->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount - nano::Gxrb_ratio) - .link (nano::dev::network_params.ledger.epochs.link (nano::epoch::epoch_1)) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*pool.generate (send->hash ())) - .build (); - auto state_send = builder - .state () - .account (nano::dev::genesis_key.pub) - .previous (epoch->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount - nano::Gxrb_ratio * 2) - .link (nano::dev::genesis_key.pub) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*pool.generate (epoch->hash ())) - .build (); - { - nano::logger_mt logger; - nano::lmdb::store store (logger, path, nano::dev::constants); - nano::stats stats; - nano::ledger ledger (store, stats, nano::dev::constants); - auto transaction (store.tx_begin_write ()); - store.initialize (transaction, ledger.cache, nano::dev::constants); - auto account_info = ledger.account_info (transaction, nano::dev::genesis->account ()); - ASSERT_TRUE (account_info); - nano::confirmation_height_info confirmation_height_info; - ASSERT_FALSE (store.confirmation_height.get (transaction, nano::dev::genesis->account (), - confirmation_height_info)); - ASSERT_EQ (confirmation_height_info.height, 1); - ASSERT_EQ (confirmation_height_info.frontier, nano::dev::genesis->hash ()); - // These databases get removed after an upgrade, so readd them - ASSERT_FALSE ( - mdb_dbi_open (store.env.tx (transaction), "state_v1", MDB_CREATE, &store.block_store.state_blocks_v1_handle)); - ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "accounts_v1", MDB_CREATE, - &store.account_store.accounts_v1_handle)); - ASSERT_FALSE ( - mdb_dbi_open (store.env.tx (transaction), "pending_v1", MDB_CREATE, &store.pending_store.pending_v1_handle)); - ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "open", MDB_CREATE, &store.block_store.open_blocks_handle)); - ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "send", MDB_CREATE, &store.block_store.send_blocks_handle)); - ASSERT_FALSE ( - mdb_dbi_open (store.env.tx (transaction), "state_blocks", MDB_CREATE, - &store.block_store.state_blocks_handle)); - ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, *send).code); - ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, *epoch).code); - ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, *state_send).code); - // Lower the database to the previous version - store.version.put (transaction, 14); - store.confirmation_height.del (transaction, nano::dev::genesis->account ()); - modify_account_info_to_v14 (store, transaction, nano::dev::genesis->account (), - confirmation_height_info.height, state_send->hash ()); - - store.pending.del (transaction, nano::pending_key (nano::dev::genesis->account (), state_send->hash ())); - - write_sideband_v14 (store, transaction, *state_send, store.block_store.state_blocks_v1_handle); - write_sideband_v14 (store, transaction, *epoch, store.block_store.state_blocks_v1_handle); - write_block_w_sideband_v18 (store, store.block_store.open_blocks_handle, transaction, *nano::dev::genesis); - write_block_w_sideband_v18 (store, store.block_store.send_blocks_handle, transaction, *send); - - // Remove from blocks table - store.block.del (transaction, state_send->hash ()); - store.block.del (transaction, epoch->hash ()); - - // Turn pending into v14 - ASSERT_FALSE (mdb_put (store.env.tx (transaction), store.pending_store.pending_v0_handle, - nano::mdb_val (nano::pending_key (nano::dev::genesis_key.pub, send->hash ())), - nano::mdb_val ( - nano::pending_info_v14 (nano::dev::genesis->account (), nano::Gxrb_ratio, - nano::epoch::epoch_0)), - 0)); - ASSERT_FALSE (mdb_put (store.env.tx (transaction), store.pending_store.pending_v1_handle, - nano::mdb_val (nano::pending_key (nano::dev::genesis_key.pub, state_send->hash ())), - nano::mdb_val ( - nano::pending_info_v14 (nano::dev::genesis->account (), nano::Gxrb_ratio, - nano::epoch::epoch_1)), - 0)); - - // This should fail as sizes are no longer correct for account_info - nano::mdb_val value; - ASSERT_FALSE (mdb_get (store.env.tx (transaction), store.account_store.accounts_v1_handle, - nano::mdb_val (nano::dev::genesis->account ()), value)); - nano::account_info info; - ASSERT_NE (value.size (), info.db_size ()); - store.account.del (transaction, nano::dev::genesis->account ()); - - // Confirmation height for the account should be deleted - ASSERT_TRUE (mdb_get (store.env.tx (transaction), store.confirmation_height_store.confirmation_height_handle, - nano::mdb_val (nano::dev::genesis->account ()), value)); - } - - // Now do the upgrade - nano::logger_mt logger; - nano::lmdb::store store (logger, path, nano::dev::constants); - ASSERT_FALSE (store.init_error ()); - auto transaction (store.tx_begin_read ()); - - // Size of account_info should now equal that set in db - nano::mdb_val value; - ASSERT_FALSE (mdb_get (store.env.tx (transaction), store.account_store.accounts_handle, - nano::mdb_val (nano::dev::genesis->account ()), value)); - nano::account_info info (value); - ASSERT_EQ (value.size (), info.db_size ()); - - // Confirmation height should exist - nano::confirmation_height_info confirmation_height_info; - ASSERT_FALSE ( - store.confirmation_height.get (transaction, nano::dev::genesis->account (), - confirmation_height_info)); - ASSERT_EQ (confirmation_height_info.height, 1); - ASSERT_EQ (confirmation_height_info.frontier, nano::dev::genesis->hash ()); - - // accounts_v1, state_blocks_v1 & pending_v1 tables should be deleted - auto error_get_accounts_v1 (mdb_get (store.env.tx (transaction), store.account_store.accounts_v1_handle, - nano::mdb_val (nano::dev::genesis->account ()), value)); - ASSERT_NE (error_get_accounts_v1, MDB_SUCCESS); - auto error_get_pending_v1 (mdb_get (store.env.tx (transaction), store.pending_store.pending_v1_handle, nano::mdb_val (nano::pending_key (nano::dev::genesis_key.pub, state_send->hash ())), value)); - ASSERT_NE (error_get_pending_v1, MDB_SUCCESS); - auto error_get_state_v1 ( - mdb_get (store.env.tx (transaction), store.block_store.state_blocks_v1_handle, nano::mdb_val (state_send->hash ()), - value)); - ASSERT_NE (error_get_state_v1, MDB_SUCCESS); - - // Check that the epochs are set correctly for the sideband, accounts and pending entries - auto block = store.block.get (transaction, state_send->hash ()); - ASSERT_NE (block, nullptr); - ASSERT_EQ (block->sideband ().details.epoch, nano::epoch::epoch_1); - block = store.block.get (transaction, send->hash ()); - ASSERT_NE (block, nullptr); - ASSERT_EQ (block->sideband ().details.epoch, nano::epoch::epoch_0); - ASSERT_EQ (info.epoch (), nano::epoch::epoch_1); - nano::pending_info pending_info; - store.pending.get (transaction, nano::pending_key (nano::dev::genesis_key.pub, send->hash ()), pending_info); - ASSERT_EQ (pending_info.epoch, nano::epoch::epoch_0); - store.pending.get (transaction, nano::pending_key (nano::dev::genesis_key.pub, state_send->hash ()), - pending_info); - ASSERT_EQ (pending_info.epoch, nano::epoch::epoch_1); - - // Version should be correct - ASSERT_LT (14, store.version.get (transaction)); + // Don't test this in rocksdb mode + GTEST_SKIP (); } - TEST (mdb_block_store, upgrade_v15_v16) - { - if (nano::rocksdb_config::using_rocksdb_in_tests ()) - { - // Don't test this in rocksdb mode - GTEST_SKIP (); - } - auto path (nano::unique_path () / "data.ldb"); - nano::mdb_val value; - { - nano::logger_mt logger; - nano::lmdb::store store (logger, path, nano::dev::constants); - nano::stats stats; - nano::ledger ledger (store, stats, nano::dev::constants); - auto transaction (store.tx_begin_write ()); - store.initialize (transaction, ledger.cache, nano::dev::constants); - // The representation table should get removed after, so readd it so that we can later confirm this actually happens - auto txn = store.env.tx (transaction); - ASSERT_FALSE ( - mdb_dbi_open (txn, "representation", MDB_CREATE, &store.account_store.representation_handle)); - auto weight = ledger.cache.rep_weights.representation_get (nano::dev::genesis->account ()); - ASSERT_EQ (MDB_SUCCESS, mdb_put (txn, store.account_store.representation_handle, nano::mdb_val (nano::dev::genesis->account ()), nano::mdb_val (nano::uint128_union (weight)), 0)); - ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "open", MDB_CREATE, &store.block_store.open_blocks_handle)); - write_block_w_sideband_v18 (store, store.block_store.open_blocks_handle, transaction, *nano::dev::genesis); - // Lower the database to the previous version - store.version.put (transaction, 15); - // Confirm the rep weight exists in the database - ASSERT_EQ (MDB_SUCCESS, mdb_get (store.env.tx (transaction), store.account_store.representation_handle, nano::mdb_val (nano::dev::genesis->account ()), value)); - store.confirmation_height.del (transaction, nano::dev::genesis->account ()); - } - - // Now do the upgrade - nano::logger_mt logger; - nano::lmdb::store store (logger, path, nano::dev::constants); - ASSERT_FALSE (store.init_error ()); - auto transaction (store.tx_begin_read ()); - - // The representation table should now be deleted - auto error_get_representation (mdb_get (store.env.tx (transaction), store.account_store.representation_handle, - nano::mdb_val (nano::dev::genesis->account ()), value)); - ASSERT_NE (MDB_SUCCESS, error_get_representation); - ASSERT_EQ (store.account_store.representation_handle, 0); + auto path (nano::unique_path () / "data.ldb"); + nano::logger_mt logger; + nano::stats stats; + auto const check_correct_state = [&] () { + nano::store::lmdb::component store (logger, path, nano::dev::constants); + auto transaction (store.tx_begin_write ()); + ASSERT_EQ (store.version.get (transaction), store.version_current); + MDB_dbi unchecked_handle{ 0 }; + ASSERT_EQ (MDB_NOTFOUND, mdb_dbi_open (store.env.tx (transaction), "unchecked", 0, &unchecked_handle)); + }; - // Version should be correct - ASSERT_LT (15, store.version.get (transaction)); - } + // Testing current version doesn't contain the unchecked table + check_correct_state (); - TEST (mdb_block_store, upgrade_v16_v17) + // Setting the database to its 21st version state { - if (nano::rocksdb_config::using_rocksdb_in_tests ()) - { - // Don't test this in rocksdb mode - GTEST_SKIP (); - } - nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits::max () }; - nano::block_builder builder; - auto block1 = builder - .state () - .account (nano::dev::genesis_key.pub) - .previous (nano::dev::genesis->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount - nano::Gxrb_ratio) - .link (nano::dev::genesis_key.pub) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*pool.generate (nano::dev::genesis->hash ())) - .build (); - auto block2 = builder - .state () - .account (nano::dev::genesis_key.pub) - .previous (block1->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount - nano::Gxrb_ratio - 1) - .link (nano::dev::genesis_key.pub) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*pool.generate (block1->hash ())) - .build (); - auto block3 = builder - .state () - .account (nano::dev::genesis_key.pub) - .previous (block2->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount - nano::Gxrb_ratio - 2) - .link (nano::dev::genesis_key.pub) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*pool.generate (block2->hash ())) - .build (); - - auto code = [&block1, &block2, &block3] (auto confirmation_height, nano::block_hash const & expected_cemented_frontier) { - auto path (nano::unique_path () / "data.ldb"); - nano::mdb_val value; - { - nano::logger_mt logger; - nano::lmdb::store store (logger, path, nano::dev::constants); - nano::stats stats; - nano::ledger ledger (store, stats, nano::dev::constants); - auto transaction (store.tx_begin_write ()); - store.initialize (transaction, ledger.cache, nano::dev::constants); - ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, *block1).code); - ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, *block2).code); - ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, *block3).code); - modify_confirmation_height_to_v15 (store, transaction, nano::dev::genesis->account (), confirmation_height); - - ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "open", MDB_CREATE, &store.block_store.open_blocks_handle)); - write_block_w_sideband_v18 (store, store.block_store.open_blocks_handle, transaction, *nano::dev::genesis); - ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "state_blocks", MDB_CREATE, &store.block_store.state_blocks_handle)); - write_block_w_sideband_v18 (store, store.block_store.state_blocks_handle, transaction, *block1); - write_block_w_sideband_v18 (store, store.block_store.state_blocks_handle, transaction, *block2); - write_block_w_sideband_v18 (store, store.block_store.state_blocks_handle, transaction, *block3); - - // Lower the database to the previous version - store.version.put (transaction, 16); - } - - // Now do the upgrade - nano::logger_mt logger; - nano::lmdb::store store (logger, path, nano::dev::constants); - ASSERT_FALSE (store.init_error ()); - auto transaction (store.tx_begin_read ()); - - nano::confirmation_height_info confirmation_height_info; - ASSERT_FALSE (store.confirmation_height.get (transaction, nano::dev::genesis->account (), confirmation_height_info)); - ASSERT_EQ (confirmation_height_info.height, confirmation_height); - - // Check confirmation height frontier is correct - ASSERT_EQ (confirmation_height_info.frontier, expected_cemented_frontier); - - // Version should be correct - ASSERT_LT (16, store.version.get (transaction)); - }; - - code (0, nano::block_hash (0)); - code (1, nano::dev::genesis->hash ()); - code (2, block1->hash ()); - code (3, block2->hash ()); - code (4, block3->hash ()); + nano::store::lmdb::component store (logger, path, nano::dev::constants); + auto transaction (store.tx_begin_write ()); + store.version.put (transaction, 21); + MDB_dbi unchecked_handle{ 0 }; + ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "unchecked", MDB_CREATE, &unchecked_handle)); + ASSERT_EQ (store.version.get (transaction), 21); } - TEST (mdb_block_store, upgrade_v17_v18) - { - if (nano::rocksdb_config::using_rocksdb_in_tests ()) - { - // Don't test this in rocksdb mode - GTEST_SKIP (); - } - auto path (nano::unique_path () / "data.ldb"); - nano::block_builder builder; - nano::keypair key1; - nano::keypair key2; - nano::keypair key3; - nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits::max () }; - auto send_zero = builder - .send () - .previous (nano::dev::genesis->hash ()) - .destination (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*pool.generate (nano::dev::genesis->hash ())) - .build (); - auto state_receive_zero = builder - .state () - .account (nano::dev::genesis_key.pub) - .previous (send_zero->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount) - .link (send_zero->hash ()) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*pool.generate (send_zero->hash ())) - .build (); - auto epoch = builder - .state () - .account (nano::dev::genesis_key.pub) - .previous (state_receive_zero->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount) - .link (nano::dev::network_params.ledger.epochs.link (nano::epoch::epoch_1)) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*pool.generate (state_receive_zero->hash ())) - .build (); - auto state_send = builder - .state () - .account (nano::dev::genesis_key.pub) - .previous (epoch->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount - nano::Gxrb_ratio) - .link (nano::dev::genesis_key.pub) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*pool.generate (epoch->hash ())) - .build (); - auto state_receive = builder - .state () - .account (nano::dev::genesis_key.pub) - .previous (state_send->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount) - .link (state_send->hash ()) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*pool.generate (state_send->hash ())) - .build (); - auto state_change = builder - .state () - .account (nano::dev::genesis_key.pub) - .previous (state_receive->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount) - .link (0) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*pool.generate (state_receive->hash ())) - .build (); - auto state_send_change = builder - .state () - .account (nano::dev::genesis_key.pub) - .previous (state_change->hash ()) - .representative (key1.pub) - .balance (nano::dev::constants.genesis_amount - nano::Gxrb_ratio) - .link (key1.pub) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*pool.generate (state_change->hash ())) - .build (); - auto epoch_first = builder - .state () - .account (key1.pub) - .previous (0) - .representative (0) - .balance (0) - .link (nano::dev::network_params.ledger.epochs.link (nano::epoch::epoch_2)) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*pool.generate (key1.pub)) - .build (); - auto state_receive2 = builder - .state () - .account (key1.pub) - .previous (epoch_first->hash ()) - .representative (key1.pub) - .balance (nano::Gxrb_ratio) - .link (state_send_change->hash ()) - .sign (key1.prv, key1.pub) - .work (*pool.generate (epoch_first->hash ())) - .build (); - auto state_send2 = builder - .state () - .account (nano::dev::genesis_key.pub) - .previous (state_send_change->hash ()) - .representative (key1.pub) - .balance (nano::dev::constants.genesis_amount - nano::Gxrb_ratio * 2) - .link (key2.pub) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*pool.generate (state_send_change->hash ())) - .build (); - auto state_open = builder - .state () - .account (key2.pub) - .previous (0) - .representative (key2.pub) - .balance (nano::Gxrb_ratio) - .link (state_send2->hash ()) - .sign (key2.prv, key2.pub) - .work (*pool.generate (key2.pub)) - .build (); - auto state_send_epoch_link = builder - .state () - .account (key2.pub) - .previous (state_open->hash ()) - .representative (key2.pub) - .balance (0) - .link (nano::dev::network_params.ledger.epochs.link (nano::epoch::epoch_2)) - .sign (key2.prv, key2.pub) - .work (*pool.generate (state_open->hash ())) - .build (); - { - nano::logger_mt logger; - nano::lmdb::store store (logger, path, nano::dev::constants); - auto transaction (store.tx_begin_write ()); - nano::stats stats; - nano::ledger ledger (store, stats, nano::dev::constants); - store.initialize (transaction, ledger.cache, nano::dev::constants); - ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, *send_zero).code); - ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, *state_receive_zero).code); - ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, *epoch).code); - ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, *state_send).code); - ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, *state_receive).code); - ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, *state_change).code); - ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, *state_send_change).code); - ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, *epoch_first).code); - ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, *state_receive2).code); - ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, *state_send2).code); - ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, *state_open).code); - ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, *state_send_epoch_link).code); - - ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "open", MDB_CREATE, &store.block_store.open_blocks_handle)); - ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "send", MDB_CREATE, &store.block_store.send_blocks_handle)); - ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "state_blocks", MDB_CREATE, &store.block_store.state_blocks_handle)); - - // Downgrade the store - store.version.put (transaction, 17); - - write_block_w_sideband_v18 (store, store.block_store.state_blocks_handle, transaction, *state_receive); - write_block_w_sideband_v18 (store, store.block_store.state_blocks_handle, transaction, *epoch_first); - write_block_w_sideband_v18 (store, store.block_store.state_blocks_handle, transaction, *state_send2); - write_block_w_sideband_v18 (store, store.block_store.state_blocks_handle, transaction, *state_send_epoch_link); - write_block_w_sideband_v18 (store, store.block_store.open_blocks_handle, transaction, *nano::dev::genesis); - write_block_w_sideband_v18 (store, store.block_store.send_blocks_handle, transaction, *send_zero); - - // Replace with the previous sideband version for state blocks - // The upgrade can resume after upgrading some blocks, test this by only downgrading some of them - write_sideband_v15 (store, transaction, *state_receive_zero); - write_sideband_v15 (store, transaction, *epoch); - write_sideband_v15 (store, transaction, *state_send); - write_sideband_v15 (store, transaction, *state_change); - write_sideband_v15 (store, transaction, *state_send_change); - write_sideband_v15 (store, transaction, *state_receive2); - write_sideband_v15 (store, transaction, *state_open); - - store.block.del (transaction, state_receive_zero->hash ()); - store.block.del (transaction, epoch->hash ()); - store.block.del (transaction, state_send->hash ()); - store.block.del (transaction, state_change->hash ()); - store.block.del (transaction, state_send_change->hash ()); - store.block.del (transaction, state_receive2->hash ()); - store.block.del (transaction, state_open->hash ()); - } - - // Now do the upgrade - nano::logger_mt logger; - nano::lmdb::store store (logger, path, nano::dev::constants); - ASSERT_FALSE (store.init_error ()); - auto transaction (store.tx_begin_read ()); - - // Size of state block should equal that set in db (no change) - nano::mdb_val value; - ASSERT_FALSE (mdb_get (store.env.tx (transaction), store.block_store.blocks_handle, nano::mdb_val (state_send->hash ()), value)); - ASSERT_EQ (value.size (), sizeof (nano::block_type) + nano::state_block::size + nano::block_sideband::size (nano::block_type::state)); - - // Check that sidebands are correctly populated - { - // Non-state unaffected - auto block = store.block.get (transaction, send_zero->hash ()); - ASSERT_NE (block, nullptr); - // All defaults - ASSERT_EQ (block->sideband ().details.epoch, nano::epoch::epoch_0); - ASSERT_FALSE (block->sideband ().details.is_epoch); - ASSERT_FALSE (block->sideband ().details.is_send); - ASSERT_FALSE (block->sideband ().details.is_receive); - } - { - // State receive from old zero send - auto block = store.block.get (transaction, state_receive_zero->hash ()); - ASSERT_NE (block, nullptr); - ASSERT_EQ (block->sideband ().details.epoch, nano::epoch::epoch_0); - ASSERT_FALSE (block->sideband ().details.is_epoch); - ASSERT_FALSE (block->sideband ().details.is_send); - ASSERT_TRUE (block->sideband ().details.is_receive); - } - { - // Epoch - auto block = store.block.get (transaction, epoch->hash ()); - ASSERT_NE (block, nullptr); - ASSERT_EQ (block->sideband ().details.epoch, nano::epoch::epoch_1); - ASSERT_TRUE (block->sideband ().details.is_epoch); - ASSERT_FALSE (block->sideband ().details.is_send); - ASSERT_FALSE (block->sideband ().details.is_receive); - } - { - // State send - auto block = store.block.get (transaction, state_send->hash ()); - ASSERT_NE (block, nullptr); - ASSERT_EQ (block->sideband ().details.epoch, nano::epoch::epoch_1); - ASSERT_FALSE (block->sideband ().details.is_epoch); - ASSERT_TRUE (block->sideband ().details.is_send); - ASSERT_FALSE (block->sideband ().details.is_receive); - } - { - // State receive - auto block = store.block.get (transaction, state_receive->hash ()); - ASSERT_NE (block, nullptr); - ASSERT_EQ (block->sideband ().details.epoch, nano::epoch::epoch_1); - ASSERT_FALSE (block->sideband ().details.is_epoch); - ASSERT_FALSE (block->sideband ().details.is_send); - ASSERT_TRUE (block->sideband ().details.is_receive); - } - { - // State change - auto block = store.block.get (transaction, state_change->hash ()); - ASSERT_NE (block, nullptr); - ASSERT_EQ (block->sideband ().details.epoch, nano::epoch::epoch_1); - ASSERT_FALSE (block->sideband ().details.is_epoch); - ASSERT_FALSE (block->sideband ().details.is_send); - ASSERT_FALSE (block->sideband ().details.is_receive); - } - { - // State send + change - auto block = store.block.get (transaction, state_send_change->hash ()); - ASSERT_NE (block, nullptr); - ASSERT_EQ (block->sideband ().details.epoch, nano::epoch::epoch_1); - ASSERT_FALSE (block->sideband ().details.is_epoch); - ASSERT_TRUE (block->sideband ().details.is_send); - ASSERT_FALSE (block->sideband ().details.is_receive); - } - { - // Epoch on unopened account - auto block = store.block.get (transaction, epoch_first->hash ()); - ASSERT_NE (block, nullptr); - ASSERT_EQ (block->sideband ().details.epoch, nano::epoch::epoch_2); - ASSERT_TRUE (block->sideband ().details.is_epoch); - ASSERT_FALSE (block->sideband ().details.is_send); - ASSERT_FALSE (block->sideband ().details.is_receive); - } - { - // State open following epoch - auto block = store.block.get (transaction, state_receive2->hash ()); - ASSERT_NE (block, nullptr); - ASSERT_EQ (block->sideband ().details.epoch, nano::epoch::epoch_2); - ASSERT_FALSE (block->sideband ().details.is_epoch); - ASSERT_FALSE (block->sideband ().details.is_send); - ASSERT_TRUE (block->sideband ().details.is_receive); - } - { - // Another state send - auto block = store.block.get (transaction, state_send2->hash ()); - ASSERT_NE (block, nullptr); - ASSERT_EQ (block->sideband ().details.epoch, nano::epoch::epoch_1); - ASSERT_FALSE (block->sideband ().details.is_epoch); - ASSERT_TRUE (block->sideband ().details.is_send); - ASSERT_FALSE (block->sideband ().details.is_receive); - } - { - // State open - auto block = store.block.get (transaction, state_open->hash ()); - ASSERT_NE (block, nullptr); - ASSERT_EQ (block->sideband ().details.epoch, nano::epoch::epoch_1); - ASSERT_FALSE (block->sideband ().details.is_epoch); - ASSERT_FALSE (block->sideband ().details.is_send); - ASSERT_TRUE (block->sideband ().details.is_receive); - } - { - // State send to an epoch link - auto block = store.block.get (transaction, state_send_epoch_link->hash ()); - ASSERT_NE (block, nullptr); - ASSERT_EQ (block->sideband ().details.epoch, nano::epoch::epoch_1); - ASSERT_FALSE (block->sideband ().details.is_epoch); - ASSERT_TRUE (block->sideband ().details.is_send); - ASSERT_FALSE (block->sideband ().details.is_receive); - } - // Version should be correct - ASSERT_LT (17, store.version.get (transaction)); - } + // Testing the upgrade code worked + check_correct_state (); +} +} - TEST (mdb_block_store, upgrade_v18_v19) +namespace nano::store::rocksdb +{ +TEST (rocksdb_block_store, upgrade_v21_v22) +{ + if (!nano::rocksdb_config::using_rocksdb_in_tests ()) { - if (nano::rocksdb_config::using_rocksdb_in_tests ()) - { - // Don't test this in rocksdb mode - GTEST_SKIP (); - } - auto path (nano::unique_path () / "data.ldb"); - nano::keypair key1; - nano::block_builder builder; - nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits::max () }; - auto send = builder - .send () - .previous (nano::dev::genesis->hash ()) - .destination (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount - nano::Gxrb_ratio) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*pool.generate (nano::dev::genesis->hash ())) - .build (); - auto receive = builder - .receive () - .previous (send->hash ()) - .source (send->hash ()) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*pool.generate (send->hash ())) - .build (); - auto change = builder - .change () - .previous (receive->hash ()) - .representative (0) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*pool.generate (receive->hash ())) - .build (); - auto state_epoch = builder - .state () - .account (nano::dev::genesis_key.pub) - .previous (change->hash ()) - .representative (0) - .balance (nano::dev::constants.genesis_amount) - .link (nano::dev::network_params.ledger.epochs.link (nano::epoch::epoch_1)) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*pool.generate (change->hash ())) - .build (); - auto state_send = builder - .state () - .account (nano::dev::genesis_key.pub) - .previous (state_epoch->hash ()) - .representative (0) - .balance (nano::dev::constants.genesis_amount - nano::Gxrb_ratio) - .link (key1.pub) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*pool.generate (state_epoch->hash ())) - .build (); - auto state_open = builder - .state () - .account (key1.pub) - .previous (0) - .representative (0) - .balance (nano::Gxrb_ratio) - .link (state_send->hash ()) - .sign (key1.prv, key1.pub) - .work (*pool.generate (key1.pub)) - .build (); - { - nano::logger_mt logger; - nano::lmdb::store store (logger, path, nano::dev::constants); - nano::stats stats; - nano::ledger ledger (store, stats, nano::dev::constants); - auto transaction (store.tx_begin_write ()); - store.initialize (transaction, ledger.cache, nano::dev::constants); - - ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, *send).code); - ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, *receive).code); - ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, *change).code); - ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, *state_epoch).code); - ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, *state_send).code); - ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, *state_open).code); - - // These tables need to be re-opened and populated so that an upgrade can be done - auto txn = store.env.tx (transaction); - ASSERT_FALSE (mdb_dbi_open (txn, "open", MDB_CREATE, &store.block_store.open_blocks_handle)); - ASSERT_FALSE (mdb_dbi_open (txn, "receive", MDB_CREATE, &store.block_store.receive_blocks_handle)); - ASSERT_FALSE (mdb_dbi_open (txn, "send", MDB_CREATE, &store.block_store.send_blocks_handle)); - ASSERT_FALSE (mdb_dbi_open (txn, "change", MDB_CREATE, &store.block_store.change_blocks_handle)); - ASSERT_FALSE (mdb_dbi_open (txn, "state_blocks", MDB_CREATE, &store.block_store.state_blocks_handle)); - - // Modify blocks back to the old tables - write_block_w_sideband_v18 (store, store.block_store.open_blocks_handle, transaction, *nano::dev::genesis); - write_block_w_sideband_v18 (store, store.block_store.send_blocks_handle, transaction, *send); - write_block_w_sideband_v18 (store, store.block_store.receive_blocks_handle, transaction, *receive); - write_block_w_sideband_v18 (store, store.block_store.change_blocks_handle, transaction, *change); - write_block_w_sideband_v18 (store, store.block_store.state_blocks_handle, transaction, *state_epoch); - write_block_w_sideband_v18 (store, store.block_store.state_blocks_handle, transaction, *state_send); - write_block_w_sideband_v18 (store, store.block_store.state_blocks_handle, transaction, *state_open); - - store.version.put (transaction, 18); - } - - // Now do the upgrade - nano::logger_mt logger; - nano::lmdb::store store (logger, path, nano::dev::constants); - ASSERT_FALSE (store.init_error ()); - auto transaction (store.tx_begin_read ()); - - // These tables should be deleted - ASSERT_EQ (store.block_store.send_blocks_handle, 0); - ASSERT_EQ (store.block_store.receive_blocks_handle, 0); - ASSERT_EQ (store.block_store.change_blocks_handle, 0); - ASSERT_EQ (store.block_store.open_blocks_handle, 0); - ASSERT_EQ (store.block_store.state_blocks_handle, 0); - - // Confirm these blocks all exist after the upgrade - ASSERT_TRUE (store.block.get (transaction, send->hash ())); - ASSERT_TRUE (store.block.get (transaction, receive->hash ())); - ASSERT_TRUE (store.block.get (transaction, change->hash ())); - ASSERT_TRUE (store.block.get (transaction, nano::dev::genesis->hash ())); - auto state_epoch_disk (store.block.get (transaction, state_epoch->hash ())); - ASSERT_NE (nullptr, state_epoch_disk); - ASSERT_EQ (nano::epoch::epoch_1, state_epoch_disk->sideband ().details.epoch); - ASSERT_EQ (nano::epoch::epoch_0, state_epoch_disk->sideband ().source_epoch); // Not used for epoch state blocks - ASSERT_TRUE (store.block.get (transaction, state_send->hash ())); - auto state_send_disk (store.block.get (transaction, state_send->hash ())); - ASSERT_NE (nullptr, state_send_disk); - ASSERT_EQ (nano::epoch::epoch_1, state_send_disk->sideband ().details.epoch); - ASSERT_EQ (nano::epoch::epoch_0, state_send_disk->sideband ().source_epoch); // Not used for send state blocks - ASSERT_TRUE (store.block.get (transaction, state_open->hash ())); - auto state_open_disk (store.block.get (transaction, state_open->hash ())); - ASSERT_NE (nullptr, state_open_disk); - ASSERT_EQ (nano::epoch::epoch_1, state_open_disk->sideband ().details.epoch); - ASSERT_EQ (nano::epoch::epoch_1, state_open_disk->sideband ().source_epoch); - - ASSERT_EQ (7, store.count (transaction, store.block_store.blocks_handle)); - - // Version should be correct - ASSERT_LT (18, store.version.get (transaction)); + // Don't test this in LMDB mode + GTEST_SKIP (); } - TEST (mdb_block_store, upgrade_v19_v20) - { - if (nano::rocksdb_config::using_rocksdb_in_tests ()) - { - // Don't test this in rocksdb mode - GTEST_SKIP (); - } - auto path (nano::unique_path () / "data.ldb"); - nano::logger_mt logger; - nano::stats stats; - { - nano::lmdb::store store (logger, path, nano::dev::constants); - nano::ledger ledger (store, stats, nano::dev::constants); - auto transaction (store.tx_begin_write ()); - store.initialize (transaction, ledger.cache, nano::dev::constants); - // Delete pruned table - ASSERT_FALSE (mdb_drop (store.env.tx (transaction), store.pruned_store.pruned_handle, 1)); - store.version.put (transaction, 19); - } - // Upgrading should create the table - nano::lmdb::store store (logger, path, nano::dev::constants); - ASSERT_FALSE (store.init_error ()); - ASSERT_NE (store.pruned_store.pruned_handle, 0); + auto const path = nano::unique_path () / "rocksdb"; + nano::logger_mt logger; + nano::stats stats; + auto const check_correct_state = [&] () { + nano::store::rocksdb::component store (logger, path, nano::dev::constants); + auto transaction (store.tx_begin_write ()); + ASSERT_EQ (store.version.get (transaction), store.version_current); + ASSERT_FALSE (store.column_family_exists ("unchecked")); + }; - // Version should be correct - auto transaction (store.tx_begin_read ()); - ASSERT_LT (19, store.version.get (transaction)); - } + // Testing current version doesn't contain the unchecked table + check_correct_state (); - TEST (mdb_block_store, upgrade_v20_v21) + // Setting the database to its 21st version state { - if (nano::rocksdb_config::using_rocksdb_in_tests ()) - { - // Don't test this in rocksdb mode - GTEST_SKIP (); - } - auto path (nano::unique_path () / "data.ldb"); - nano::logger_mt logger; - nano::stats stats; - { - nano::lmdb::store store (logger, path, nano::dev::constants); - nano::ledger ledger (store, stats, nano::dev::constants); - auto transaction (store.tx_begin_write ()); - store.initialize (transaction, ledger.cache, ledger.constants); - // Delete pruned table - ASSERT_FALSE (mdb_drop (store.env.tx (transaction), store.final_vote_store.final_votes_handle, 1)); - store.version.put (transaction, 20); - } - // Upgrading should create the table - nano::lmdb::store store (logger, path, nano::dev::constants); - ASSERT_FALSE (store.init_error ()); - ASSERT_NE (store.final_vote_store.final_votes_handle, 0); + nano::store::rocksdb::component store (logger, path, nano::dev::constants); - // Version should be correct - auto transaction (store.tx_begin_read ()); - ASSERT_LT (19, store.version.get (transaction)); - } + // Create a column family for "unchecked" + ::rocksdb::ColumnFamilyOptions new_cf_options; + ::rocksdb::ColumnFamilyHandle * new_cf_handle; + ::rocksdb::Status status = store.db->CreateColumnFamily (new_cf_options, "unchecked", &new_cf_handle); + store.handles.emplace_back (new_cf_handle); - TEST (mdb_block_store, upgrade_v21_v22) - { - if (nano::rocksdb_config::using_rocksdb_in_tests ()) - { - // Don't test this in rocksdb mode - GTEST_SKIP (); - } + // The new column family was created successfully, and 'new_cf_handle' now points to it. + ASSERT_TRUE (status.ok ()); - auto path (nano::unique_path () / "data.ldb"); - nano::logger_mt logger; - nano::stats stats; - auto const check_correct_state = [&] () { - nano::lmdb::store store (logger, path, nano::dev::constants); - auto transaction (store.tx_begin_write ()); - ASSERT_EQ (store.version.get (transaction), store.version_current); - MDB_dbi unchecked_handle{ 0 }; - ASSERT_EQ (MDB_NOTFOUND, mdb_dbi_open (store.env.tx (transaction), "unchecked", 0, &unchecked_handle)); - }; - - // Testing current version doesn't contain the unchecked table - check_correct_state (); - - // Setting the database to its 21st version state - { - nano::lmdb::store store (logger, path, nano::dev::constants); - auto transaction (store.tx_begin_write ()); - store.version.put (transaction, 21); - MDB_dbi unchecked_handle{ 0 }; - ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "unchecked", MDB_CREATE, &unchecked_handle)); - ASSERT_EQ (store.version.get (transaction), 21); - } - - // Testing the upgrade code worked - check_correct_state (); + // Rollback the database version number. + auto transaction (store.tx_begin_write ()); + store.version.put (transaction, 21); + ASSERT_EQ (store.version.get (transaction), 21); } -} - -namespace rocksdb -{ - TEST (rocksdb_block_store, upgrade_v21_v22) - { - if (!nano::rocksdb_config::using_rocksdb_in_tests ()) - { - // Don't test this in LMDB mode - GTEST_SKIP (); - } - - auto const path = nano::unique_path () / "rocksdb"; - nano::logger_mt logger; - nano::stats stats; - auto const check_correct_state = [&] () { - nano::rocksdb::store store (logger, path, nano::dev::constants); - auto transaction (store.tx_begin_write ()); - ASSERT_EQ (store.version.get (transaction), store.version_current); - ASSERT_FALSE (store.column_family_exists ("unchecked")); - }; - - // Testing current version doesn't contain the unchecked table - check_correct_state (); - - // Setting the database to its 21st version state - { - nano::rocksdb::store store (logger, path, nano::dev::constants); - - // Create a column family for "unchecked" - ::rocksdb::ColumnFamilyOptions new_cf_options; - ::rocksdb::ColumnFamilyHandle * new_cf_handle; - ::rocksdb::Status status = store.db->CreateColumnFamily (new_cf_options, "unchecked", &new_cf_handle); - store.handles.emplace_back (new_cf_handle); - - // The new column family was created successfully, and 'new_cf_handle' now points to it. - ASSERT_TRUE (status.ok ()); - - // Rollback the database version number. - auto transaction (store.tx_begin_write ()); - store.version.put (transaction, 21); - ASSERT_EQ (store.version.get (transaction), 21); - } - // Testing the upgrade code worked - check_correct_state (); - } + // Testing the upgrade code worked + check_correct_state (); } } @@ -2288,15 +1462,15 @@ TEST (mdb_block_store, upgrade_backup) { nano::logger_mt logger; - nano::lmdb::store store (logger, path, nano::dev::constants); + nano::store::lmdb::component store (logger, path, nano::dev::constants); auto transaction (store.tx_begin_write ()); - store.version.put (transaction, 14); + store.version.put (transaction, store.version_minimum); } ASSERT_EQ (get_backup_path ().string (), dir.string ()); // Now do the upgrade and confirm that backup is saved nano::logger_mt logger; - nano::lmdb::store store (logger, path, nano::dev::constants, nano::txn_tracking_config{}, std::chrono::seconds (5), nano::lmdb_config{}, true); + nano::store::lmdb::component store (logger, path, nano::dev::constants, nano::txn_tracking_config{}, std::chrono::seconds (5), nano::lmdb_config{}, true); ASSERT_FALSE (store.init_error ()); auto transaction (store.tx_begin_read ()); ASSERT_LT (14, store.version.get (transaction)); @@ -2454,10 +1628,10 @@ TEST (block_store, rocksdb_force_test_env_variable) auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants); - auto mdb_cast = dynamic_cast (store.get ()); + auto mdb_cast = dynamic_cast (store.get ()); if (value && boost::lexical_cast (value) == 1) { - ASSERT_NE (boost::polymorphic_downcast (store.get ()), nullptr); + ASSERT_NE (boost::polymorphic_downcast (store.get ()), nullptr); } else { @@ -2477,7 +1651,7 @@ TEST (rocksdb_block_store, tombstone_count) } nano::test::system system; nano::logger_mt logger; - auto store = std::make_unique (logger, nano::unique_path () / "rocksdb", nano::dev::constants); + auto store = std::make_unique (logger, nano::unique_path () / "rocksdb", nano::dev::constants); ASSERT_TRUE (!store->init_error ()); nano::block_builder builder; auto block = builder @@ -2491,7 +1665,7 @@ TEST (rocksdb_block_store, tombstone_count) // Enqueues a block to be saved in the database nano::account account{ 1 }; store->account.put (store->tx_begin_write (), account, nano::account_info{}); - auto check_block_is_listed = [&] (nano::transaction const & transaction_a) { + auto check_block_is_listed = [&] (store::transaction const & transaction_a) { return store->account.exists (transaction_a, account); }; // Waits for the block to get saved @@ -2502,79 +1676,3 @@ TEST (rocksdb_block_store, tombstone_count) ASSERT_TIMELY (5s, store->tombstone_map.at (nano::tables::accounts).num_since_last_flush.load () == 1); } } - -namespace nano -{ -namespace lmdb -{ - void write_sideband_v14 (nano::lmdb::store & store_a, nano::transaction & transaction_a, nano::block const & block_a, MDB_dbi db_a) - { - auto block = store_a.block.get (transaction_a, block_a.hash ()); - ASSERT_NE (block, nullptr); - - nano::block_sideband_v14 sideband_v14 (block->type (), block->sideband ().account, block->sideband ().successor, block->sideband ().balance, block->sideband ().timestamp, block->sideband ().height); - std::vector data; - { - nano::vectorstream stream (data); - block_a.serialize (stream); - sideband_v14.serialize (stream); - } - - MDB_val val{ data.size (), data.data () }; - ASSERT_FALSE (mdb_put (store_a.env.tx (transaction_a), block->sideband ().details.epoch == nano::epoch::epoch_0 ? store_a.block_store.state_blocks_v0_handle : store_a.block_store.state_blocks_v1_handle, nano::mdb_val (block_a.hash ()), &val, 0)); - } - - void write_sideband_v15 (nano::lmdb::store & store_a, nano::transaction & transaction_a, nano::block const & block_a) - { - auto block = store_a.block.get (transaction_a, block_a.hash ()); - ASSERT_NE (block, nullptr); - - ASSERT_LE (block->sideband ().details.epoch, nano::epoch::max); - // Simulated by writing 0 on every of the most significant bits, leaving out epoch only, as if pre-upgrade - nano::block_sideband_v18 sideband_v15 (block->sideband ().account, block->sideband ().successor, block->sideband ().balance, block->sideband ().timestamp, block->sideband ().height, block->sideband ().details.epoch, false, false, false); - std::vector data; - { - nano::vectorstream stream (data); - block_a.serialize (stream); - sideband_v15.serialize (stream, block_a.type ()); - } - - MDB_val val{ data.size (), data.data () }; - ASSERT_FALSE (mdb_put (store_a.env.tx (transaction_a), store_a.block_store.state_blocks_handle, nano::mdb_val (block_a.hash ()), &val, 0)); - } - - void write_block_w_sideband_v18 (nano::lmdb::store & store_a, MDB_dbi database, nano::write_transaction & transaction_a, nano::block const & block_a) - { - auto block = store_a.block.get (transaction_a, block_a.hash ()); - ASSERT_NE (block, nullptr); - auto new_sideband (block->sideband ()); - nano::block_sideband_v18 sideband_v18 (new_sideband.account, new_sideband.successor, new_sideband.balance, new_sideband.height, new_sideband.timestamp, new_sideband.details.epoch, new_sideband.details.is_send, new_sideband.details.is_receive, new_sideband.details.is_epoch); - - std::vector data; - { - nano::vectorstream stream (data); - block->serialize (stream); - sideband_v18.serialize (stream, block->type ()); - } - - MDB_val val{ data.size (), data.data () }; - ASSERT_FALSE (mdb_put (store_a.env.tx (transaction_a), database, nano::mdb_val (block_a.hash ()), &val, 0)); - store_a.del (transaction_a, nano::tables::blocks, nano::mdb_val (block_a.hash ())); - } - - void modify_account_info_to_v14 (nano::lmdb::store & store, nano::transaction const & transaction, nano::account const & account, uint64_t confirmation_height, nano::block_hash const & rep_block) - { - nano::account_info info; - ASSERT_FALSE (store.account.get (transaction, account, info)); - nano::account_info_v14 account_info_v14 (info.head, rep_block, info.open_block, info.balance, info.modified, info.block_count, confirmation_height, info.epoch ()); - auto status (mdb_put (store.env.tx (transaction), info.epoch () == nano::epoch::epoch_0 ? store.account_store.accounts_v0_handle : store.account_store.accounts_v1_handle, nano::mdb_val (account), nano::mdb_val (account_info_v14), 0)); - ASSERT_EQ (status, 0); - } - - void modify_confirmation_height_to_v15 (nano::lmdb::store & store, nano::transaction const & transaction, nano::account const & account, uint64_t confirmation_height) - { - auto status (mdb_put (store.env.tx (transaction), store.confirmation_height_store.confirmation_height_handle, nano::mdb_val (account), nano::mdb_val (confirmation_height), 0)); - ASSERT_EQ (status, 0); - } -} -} diff --git a/nano/core_test/bootstrap_ascending.cpp b/nano/core_test/bootstrap_ascending.cpp index cdc2e5c38e..fd3850ccee 100644 --- a/nano/core_test/bootstrap_ascending.cpp +++ b/nano/core_test/bootstrap_ascending.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include diff --git a/nano/core_test/confirmation_height.cpp b/nano/core_test/confirmation_height.cpp index 508697f4ee..7a014447ef 100644 --- a/nano/core_test/confirmation_height.cpp +++ b/nano/core_test/confirmation_height.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -390,7 +391,7 @@ TEST (confirmation_height, gap_bootstrap) // Receive 2 comes in on the live network, however the chain has not been finished so it gets added to unchecked node1.process_active (receive2); // Waits for the unchecked_map to process the 4 blocks added to the block_processor, saving them in the unchecked table - auto check_block_is_listed = [&] (nano::transaction const & transaction_a, nano::block_hash const & block_hash_a) { + auto check_block_is_listed = [&] (nano::store::transaction const & transaction_a, nano::block_hash const & block_hash_a) { return !node1.unchecked.get (block_hash_a).empty (); }; ASSERT_TIMELY (5s, check_block_is_listed (node1.store.tx_begin_read (), receive2->previous ())); diff --git a/nano/core_test/gap_cache.cpp b/nano/core_test/gap_cache.cpp index e0554f61ab..70e1b0ac94 100644 --- a/nano/core_test/gap_cache.cpp +++ b/nano/core_test/gap_cache.cpp @@ -1,3 +1,4 @@ +#include #include #include diff --git a/nano/core_test/ledger.cpp b/nano/core_test/ledger.cpp index 52e7c289fa..b2d08ba9cf 100644 --- a/nano/core_test/ledger.cpp +++ b/nano/core_test/ledger.cpp @@ -1,10 +1,11 @@ #include #include #include -#include +#include #include #include #include +#include #include #include #include @@ -112,7 +113,7 @@ TEST (ledger, process_send) ASSERT_TRUE (store.frontier.get (transaction, info1->head).is_zero ()); ASSERT_EQ (nano::dev::genesis_key.pub, store.frontier.get (transaction, hash1)); ASSERT_EQ (nano::process_result::progress, return1.code); - ASSERT_EQ (nano::dev::genesis_key.pub, store.block.account_calculated (*send)); + ASSERT_EQ (nano::dev::genesis_key.pub, ledger.account (*send)); ASSERT_EQ (50, ledger.account_balance (transaction, nano::dev::genesis_key.pub)); ASSERT_EQ (nano::dev::constants.genesis_amount - 50, ledger.account_receivable (transaction, key2.pub)); auto info2 = ledger.account_info (transaction, nano::dev::genesis_key.pub); @@ -141,7 +142,7 @@ TEST (ledger, process_send) ASSERT_EQ (1, open->sideband ().height); ASSERT_EQ (nano::dev::constants.genesis_amount - 50, ledger.amount (transaction, hash2)); ASSERT_EQ (nano::process_result::progress, return2.code); - ASSERT_EQ (key2.pub, store.block.account_calculated (*open)); + ASSERT_EQ (key2.pub, ledger.account (*open)); ASSERT_EQ (nano::dev::constants.genesis_amount - 50, ledger.amount (transaction, hash2)); ASSERT_EQ (key2.pub, store.frontier.get (transaction, hash2)); ASSERT_EQ (nano::dev::constants.genesis_amount - 50, ledger.account_balance (transaction, key2.pub)); @@ -225,7 +226,7 @@ TEST (ledger, process_receive) nano::block_hash hash2 (open->hash ()); auto return1 = ledger.process (transaction, *open); ASSERT_EQ (nano::process_result::progress, return1.code); - ASSERT_EQ (key2.pub, store.block.account_calculated (*open)); + ASSERT_EQ (key2.pub, ledger.account (*open)); ASSERT_EQ (key2.pub, open->sideband ().account); ASSERT_EQ (nano::dev::constants.genesis_amount - 50, open->sideband ().balance.number ()); ASSERT_EQ (1, open->sideband ().height); @@ -258,7 +259,7 @@ TEST (ledger, process_receive) ASSERT_TRUE (store.frontier.get (transaction, hash2).is_zero ()); ASSERT_EQ (key2.pub, store.frontier.get (transaction, hash4)); ASSERT_EQ (nano::process_result::progress, return2.code); - ASSERT_EQ (key2.pub, store.block.account_calculated (*receive)); + ASSERT_EQ (key2.pub, ledger.account (*receive)); ASSERT_EQ (hash4, ledger.latest (transaction, key2.pub)); ASSERT_EQ (25, ledger.account_balance (transaction, nano::dev::genesis_key.pub)); ASSERT_EQ (0, ledger.account_receivable (transaction, key2.pub)); @@ -525,7 +526,7 @@ TEST (ledger, representative_change) ASSERT_TRUE (store.frontier.get (transaction, info1->head).is_zero ()); ASSERT_EQ (nano::dev::genesis_key.pub, store.frontier.get (transaction, block->hash ())); ASSERT_EQ (nano::process_result::progress, return1.code); - ASSERT_EQ (nano::dev::genesis_key.pub, store.block.account_calculated (*block)); + ASSERT_EQ (nano::dev::genesis_key.pub, ledger.account (*block)); ASSERT_EQ (0, ledger.weight (nano::dev::genesis_key.pub)); ASSERT_EQ (nano::dev::constants.genesis_amount, ledger.weight (key2.pub)); auto info2 = ledger.account_info (transaction, nano::dev::genesis_key.pub); @@ -4494,7 +4495,7 @@ TEST (ledger, unchecked_receive) node1.work_generate_blocking (*receive1); node1.block_processor.add (send1); node1.block_processor.add (receive1); - auto check_block_is_listed = [&] (nano::transaction const & transaction_a, nano::block_hash const & block_hash_a) { + auto check_block_is_listed = [&] (nano::store::transaction const & transaction_a, nano::block_hash const & block_hash_a) { return !node1.unchecked.get (block_hash_a).empty (); }; // Previous block for receive1 is unknown, signature cannot be validated @@ -5542,7 +5543,7 @@ TEST (ledger, migrate_lmdb_to_rocksdb) nano::logger_mt logger{}; boost::asio::ip::address_v6 address (boost::asio::ip::make_address_v6 ("::ffff:127.0.0.1")); uint16_t port = 100; - nano::lmdb::store store{ logger, path / "data.ldb", nano::dev::constants }; + nano::store::lmdb::component store{ logger, path / "data.ldb", nano::dev::constants }; nano::ledger ledger{ store, system.stats, nano::dev::constants }; nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits::max () }; @@ -5557,7 +5558,7 @@ TEST (ledger, migrate_lmdb_to_rocksdb) .build_shared (); nano::endpoint_key endpoint_key (address.to_bytes (), port); - auto version = nano::store::version_current; + auto version = nano::store::component::version_current; { auto transaction = store.tx_begin_write (); @@ -5582,7 +5583,7 @@ TEST (ledger, migrate_lmdb_to_rocksdb) auto error = ledger.migrate_lmdb_to_rocksdb (path); ASSERT_FALSE (error); - nano::rocksdb::store rocksdb_store{ logger, path / "rocksdb", nano::dev::constants }; + nano::store::rocksdb::component rocksdb_store{ logger, path / "rocksdb", nano::dev::constants }; auto rocksdb_transaction (rocksdb_store.tx_begin_read ()); nano::pending_info pending_info{}; diff --git a/nano/core_test/ledger_walker.cpp b/nano/core_test/ledger_walker.cpp index ccedb16fc9..29797a212d 100644 --- a/nano/core_test/ledger_walker.cpp +++ b/nano/core_test/ledger_walker.cpp @@ -111,11 +111,11 @@ TEST (ledger_walker, cross_account) ASSERT_TRUE (account_info); // TODO: check issue with account head - // auto const first = node->ledger.store.block.get_no_sideband(transaction, account_info.head); - // auto const second = node->ledger.store.block.get_no_sideband(transaction, first->previous()); - // auto const third = node->ledger.store.block.get_no_sideband(transaction, second->previous()); - // auto const fourth = node->ledger.store.block.get_no_sideband(transaction, third->previous()); - // auto const fifth = node->ledger.store.block.get_no_sideband(transaction, fourth->previous()); + // auto const first = node->ledger.store.block.get(transaction, account_info.head); + // auto const second = node->ledger.store.block.get(transaction, first->previous()); + // auto const third = node->ledger.store.block.get(transaction, second->previous()); + // auto const fourth = node->ledger.store.block.get(transaction, third->previous()); + // auto const fifth = node->ledger.store.block.get(transaction, fourth->previous()); // // auto const expected_blocks_to_walk = { first, second, third, fourth, fifth }; // auto expected_blocks_to_walk_itr = expected_blocks_to_walk.begin(); @@ -193,7 +193,7 @@ TEST (ledger_walker, DISABLED_ladder_geometry) nano::amount previous_balance{}; if (!block->previous ().is_zero ()) { - auto const previous_block = node->ledger.store.block.get_no_sideband (node->ledger.store.tx_begin_read (), block->previous ()); + auto const previous_block = node->ledger.store.block.get (node->ledger.store.tx_begin_read (), block->previous ()); previous_balance = previous_block->balance (); } @@ -212,7 +212,7 @@ TEST (ledger_walker, DISABLED_ladder_geometry) nano::amount previous_balance{}; if (!block->previous ().is_zero ()) { - auto const previous_block = node->ledger.store.block.get_no_sideband (node->ledger.store.tx_begin_read (), block->previous ()); + auto const previous_block = node->ledger.store.block.get (node->ledger.store.tx_begin_read (), block->previous ()); previous_balance = previous_block->balance (); } diff --git a/nano/core_test/logger.cpp b/nano/core_test/logger.cpp index d60cf397df..88bccc5a4d 100644 --- a/nano/core_test/logger.cpp +++ b/nano/core_test/logger.cpp @@ -5,6 +5,8 @@ #include +#include + #include #include #include diff --git a/nano/core_test/node.cpp b/nano/core_test/node.cpp index ab4408a53d..c445d0a280 100644 --- a/nano/core_test/node.cpp +++ b/nano/core_test/node.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include diff --git a/nano/core_test/processor_service.cpp b/nano/core_test/processor_service.cpp index c873a80dc7..42893a97d0 100644 --- a/nano/core_test/processor_service.cpp +++ b/nano/core_test/processor_service.cpp @@ -1,8 +1,9 @@ #include #include +#include #include -#include #include +#include #include #include diff --git a/nano/core_test/socket.cpp b/nano/core_test/socket.cpp index d0b126dafe..ea716bb3e1 100644 --- a/nano/core_test/socket.cpp +++ b/nano/core_test/socket.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include diff --git a/nano/core_test/system.cpp b/nano/core_test/system.cpp index 2673d4de1d..41e7f9c841 100644 --- a/nano/core_test/system.cpp +++ b/nano/core_test/system.cpp @@ -1,3 +1,4 @@ +#include #include #include #include diff --git a/nano/core_test/toml.cpp b/nano/core_test/toml.cpp index eb3f60b46d..b8098dd6ea 100644 --- a/nano/core_test/toml.cpp +++ b/nano/core_test/toml.cpp @@ -8,6 +8,8 @@ #include +#include + #include #include #include diff --git a/nano/core_test/utility.cpp b/nano/core_test/utility.cpp index 3d418e91a7..652ea8e486 100644 --- a/nano/core_test/utility.cpp +++ b/nano/core_test/utility.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #include #include #include diff --git a/nano/core_test/wallet.cpp b/nano/core_test/wallet.cpp index 799466e2ab..6aad11bc15 100644 --- a/nano/core_test/wallet.cpp +++ b/nano/core_test/wallet.cpp @@ -1,6 +1,7 @@ #include +#include #include -#include +#include #include #include @@ -14,7 +15,7 @@ unsigned constexpr nano::wallet_store::version_current; TEST (wallet, no_special_keys_accounts) { bool init; - nano::mdb_env env (init, nano::unique_path ()); + nano::store::lmdb::env env (init, nano::unique_path ()); ASSERT_FALSE (init); auto transaction (env.tx_begin_write ()); nano::kdf kdf{ nano::dev::network_params.kdf_work }; @@ -35,7 +36,7 @@ TEST (wallet, no_special_keys_accounts) TEST (wallet, no_key) { bool init; - nano::mdb_env env (init, nano::unique_path ()); + nano::store::lmdb::env env (init, nano::unique_path ()); ASSERT_FALSE (init); auto transaction (env.tx_begin_write ()); nano::kdf kdf{ nano::dev::network_params.kdf_work }; @@ -50,7 +51,7 @@ TEST (wallet, no_key) TEST (wallet, fetch_locked) { bool init; - nano::mdb_env env (init, nano::unique_path ()); + nano::store::lmdb::env env (init, nano::unique_path ()); ASSERT_FALSE (init); auto transaction (env.tx_begin_write ()); nano::kdf kdf{ nano::dev::network_params.kdf_work }; @@ -72,7 +73,7 @@ TEST (wallet, fetch_locked) TEST (wallet, retrieval) { bool init; - nano::mdb_env env (init, nano::unique_path ()); + nano::store::lmdb::env env (init, nano::unique_path ()); ASSERT_FALSE (init); auto transaction (env.tx_begin_write ()); nano::kdf kdf{ nano::dev::network_params.kdf_work }; @@ -94,7 +95,7 @@ TEST (wallet, retrieval) TEST (wallet, empty_iteration) { bool init; - nano::mdb_env env (init, nano::unique_path ()); + nano::store::lmdb::env env (init, nano::unique_path ()); ASSERT_FALSE (init); auto transaction (env.tx_begin_write ()); nano::kdf kdf{ nano::dev::network_params.kdf_work }; @@ -108,7 +109,7 @@ TEST (wallet, empty_iteration) TEST (wallet, one_item_iteration) { bool init; - nano::mdb_env env (init, nano::unique_path ()); + nano::store::lmdb::env env (init, nano::unique_path ()); ASSERT_FALSE (init); auto transaction (env.tx_begin_write ()); nano::kdf kdf{ nano::dev::network_params.kdf_work }; @@ -130,7 +131,7 @@ TEST (wallet, one_item_iteration) TEST (wallet, two_item_iteration) { bool init; - nano::mdb_env env (init, nano::unique_path ()); + nano::store::lmdb::env env (init, nano::unique_path ()); ASSERT_FALSE (init); nano::keypair key1; nano::keypair key2; @@ -269,7 +270,7 @@ TEST (wallet, spend_no_previous) TEST (wallet, find_none) { bool init; - nano::mdb_env env (init, nano::unique_path ()); + nano::store::lmdb::env env (init, nano::unique_path ()); ASSERT_FALSE (init); auto transaction (env.tx_begin_write ()); nano::kdf kdf{ nano::dev::network_params.kdf_work }; @@ -282,7 +283,7 @@ TEST (wallet, find_none) TEST (wallet, find_existing) { bool init; - nano::mdb_env env (init, nano::unique_path ()); + nano::store::lmdb::env env (init, nano::unique_path ()); ASSERT_FALSE (init); auto transaction (env.tx_begin_write ()); nano::kdf kdf{ nano::dev::network_params.kdf_work }; @@ -301,7 +302,7 @@ TEST (wallet, find_existing) TEST (wallet, rekey) { bool init; - nano::mdb_env env (init, nano::unique_path ()); + nano::store::lmdb::env env (init, nano::unique_path ()); ASSERT_FALSE (init); auto transaction (env.tx_begin_write ()); nano::kdf kdf{ nano::dev::network_params.kdf_work }; @@ -373,7 +374,7 @@ TEST (account, encode_fail) TEST (wallet, hash_password) { bool init; - nano::mdb_env env (init, nano::unique_path ()); + nano::store::lmdb::env env (init, nano::unique_path ()); ASSERT_FALSE (init); auto transaction (env.tx_begin_write ()); nano::kdf kdf{ nano::dev::network_params.kdf_work }; @@ -422,7 +423,7 @@ TEST (fan, change) TEST (wallet, reopen_default_password) { bool init; - nano::mdb_env env (init, nano::unique_path ()); + nano::store::lmdb::env env (init, nano::unique_path ()); auto transaction (env.tx_begin_write ()); ASSERT_FALSE (init); nano::kdf kdf{ nano::dev::network_params.kdf_work }; @@ -458,7 +459,7 @@ TEST (wallet, reopen_default_password) TEST (wallet, representative) { auto error (false); - nano::mdb_env env (error, nano::unique_path ()); + nano::store::lmdb::env env (error, nano::unique_path ()); ASSERT_FALSE (error); auto transaction (env.tx_begin_write ()); nano::kdf kdf{ nano::dev::network_params.kdf_work }; @@ -479,7 +480,7 @@ TEST (wallet, representative) TEST (wallet, serialize_json_empty) { auto error (false); - nano::mdb_env env (error, nano::unique_path ()); + nano::store::lmdb::env env (error, nano::unique_path ()); ASSERT_FALSE (error); auto transaction (env.tx_begin_write ()); nano::kdf kdf{ nano::dev::network_params.kdf_work }; @@ -504,7 +505,7 @@ TEST (wallet, serialize_json_empty) TEST (wallet, serialize_json_one) { auto error (false); - nano::mdb_env env (error, nano::unique_path ()); + nano::store::lmdb::env env (error, nano::unique_path ()); ASSERT_FALSE (error); auto transaction (env.tx_begin_write ()); nano::kdf kdf{ nano::dev::network_params.kdf_work }; @@ -533,7 +534,7 @@ TEST (wallet, serialize_json_one) TEST (wallet, serialize_json_password) { auto error (false); - nano::mdb_env env (error, nano::unique_path ()); + nano::store::lmdb::env env (error, nano::unique_path ()); ASSERT_FALSE (error); auto transaction (env.tx_begin_write ()); nano::kdf kdf{ nano::dev::network_params.kdf_work }; @@ -566,7 +567,7 @@ TEST (wallet, serialize_json_password) TEST (wallet_store, move) { auto error (false); - nano::mdb_env env (error, nano::unique_path ()); + nano::store::lmdb::env env (error, nano::unique_path ()); ASSERT_FALSE (error); auto transaction (env.tx_begin_write ()); nano::kdf kdf{ nano::dev::network_params.kdf_work }; @@ -724,7 +725,7 @@ TEST (wallet, insert_locked) TEST (wallet, deterministic_keys) { bool init; - nano::mdb_env env (init, nano::unique_path ()); + nano::store::lmdb::env env (init, nano::unique_path ()); ASSERT_FALSE (init); auto transaction (env.tx_begin_write ()); nano::kdf kdf{ nano::dev::network_params.kdf_work }; @@ -767,7 +768,7 @@ TEST (wallet, deterministic_keys) TEST (wallet, reseed) { bool init; - nano::mdb_env env (init, nano::unique_path ()); + nano::store::lmdb::env env (init, nano::unique_path ()); ASSERT_FALSE (init); auto transaction (env.tx_begin_write ()); nano::kdf kdf{ nano::dev::network_params.kdf_work }; @@ -1072,7 +1073,7 @@ TEST (wallet, epoch_2_receive_propagation) if (nano::dev::network_params.work.difficulty (*receive2) < node.network_params.work.base) { ASSERT_GE (nano::dev::network_params.work.difficulty (*receive2), node.network_params.work.epoch_2_receive); - ASSERT_EQ (nano::epoch::epoch_2, node.store.block.version (node.store.tx_begin_read (), receive2->hash ())); + ASSERT_EQ (nano::epoch::epoch_2, node.ledger.version (*receive2)); ASSERT_EQ (nano::epoch::epoch_2, receive2->sideband ().source_epoch); break; } @@ -1125,7 +1126,7 @@ TEST (wallet, epoch_2_receive_unopened) if (nano::dev::network_params.work.difficulty (*receive1) < node.network_params.work.base) { ASSERT_GE (nano::dev::network_params.work.difficulty (*receive1), node.network_params.work.epoch_2_receive); - ASSERT_EQ (nano::epoch::epoch_2, node.store.block.version (node.store.tx_begin_read (), receive1->hash ())); + ASSERT_EQ (nano::epoch::epoch_2, node.ledger.version (*receive1)); ASSERT_EQ (nano::epoch::epoch_1, receive1->sideband ().source_epoch); break; } diff --git a/nano/core_test/wallets.cpp b/nano/core_test/wallets.cpp index ad9e6df496..6f6999329c 100644 --- a/nano/core_test/wallets.cpp +++ b/nano/core_test/wallets.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include @@ -73,7 +73,9 @@ TEST (wallets, remove) } } -TEST (wallets, reload) +// Opening multiple environments using the same file within the same process is not supported. +// http://www.lmdb.tech/doc/starting.html +TEST (wallets, DISABLED_reload) { nano::test::system system (1); auto & node1 (*system.nodes[0]); diff --git a/nano/ipc_flatbuffers_lib/CMakeLists.txt b/nano/ipc_flatbuffers_lib/CMakeLists.txt index 9b0d2e5685..f5323c47e5 100644 --- a/nano/ipc_flatbuffers_lib/CMakeLists.txt +++ b/nano/ipc_flatbuffers_lib/CMakeLists.txt @@ -46,7 +46,8 @@ file(GLOB files foreach(file ${files}) get_filename_component(flatbuffers_filename ${file} NAME_WE) message( - "Generating flatbuffers code for: ${flatbuffers_filename} into ${CMAKE_CURRENT_SOURCE_DIR}/generated/flatbuffers" + STATUS + "Generating flatbuffers code for: ${flatbuffers_filename} into ${CMAKE_CURRENT_SOURCE_DIR}/generated/flatbuffers" ) configure_file( diff --git a/nano/lib/CMakeLists.txt b/nano/lib/CMakeLists.txt index 8e24640dc9..4acaf59bb6 100644 --- a/nano/lib/CMakeLists.txt +++ b/nano/lib/CMakeLists.txt @@ -75,8 +75,12 @@ add_library( stats_enums.hpp stats_enums.cpp stream.hpp + thread_pool.hpp + thread_pool.cpp thread_roles.hpp thread_roles.cpp + thread_runner.hpp + thread_runner.cpp threading.hpp threading.cpp timer.hpp diff --git a/nano/lib/errors.cpp b/nano/lib/errors.cpp index f4eabbd6ac..822722be62 100644 --- a/nano/lib/errors.cpp +++ b/nano/lib/errors.cpp @@ -282,48 +282,11 @@ std::string nano::error_config_messages::message (int ev) const return "Invalid error code"; } -#if defined(NANO_USE_BOOST_TO_STD_ERROR_BRIDGE) -char const * nano::error_conversion::detail::generic_category::name () const noexcept -{ - return boost::system::generic_category ().name (); -} - -std::string nano::error_conversion::detail::generic_category::message (int value) const -{ - return boost::system::generic_category ().message (value); -} - -std::error_category const & nano::error_conversion::generic_category () -{ - static detail::generic_category instance; - return instance; -} - -std::error_code nano::error_conversion::convert (boost::system::error_code const & error) -{ - if (error.category () == boost::system::generic_category ()) - { - return std::error_code (error.value (), - nano::error_conversion::generic_category ()); - } - - debug_assert (false); - return nano::error_common::invalid_type_conversion; -} -#endif - nano::error::error (std::error_code code_a) { code = code_a; } -#if defined(NANO_USE_BOOST_TO_STD_ERROR_BRIDGE) -nano::error::error (boost::system::error_code const & code_a) -{ - code = std::make_error_code (static_cast (code_a.value ())); -} -#endif - nano::error::error (std::string message_a) { code = nano::error_common::generic; @@ -358,24 +321,6 @@ nano::error & nano::error::operator= (std::error_code const code_a) return *this; } -#if defined(NANO_USE_BOOST_TO_STD_ERROR_BRIDGE) -/** Assign boost error code (as converted to std::error_code) */ -nano::error & nano::error::operator= (boost::system::error_code const & code_a) -{ - code = nano::error_conversion::convert (code_a); - message.clear (); - return *this; -} - -/** Assign boost error code (as converted to std::error_code) */ -nano::error & nano::error::operator= (boost::system::errc::errc_t const & code_a) -{ - code = nano::error_conversion::convert (boost::system::errc::make_error_code (code_a)); - message.clear (); - return *this; -} -#endif - /** Set the error to nano::error_common::generic and the error message to \p message_a */ nano::error & nano::error::operator= (std::string message_a) { @@ -398,14 +343,6 @@ bool nano::error::operator== (std::error_code const code_a) const return code == code_a; } -#if defined(NANO_USE_BOOST_TO_STD_ERROR_BRIDGE) -/** Return true if this#error_code equals the parameter */ -bool nano::error::operator== (boost::system::error_code const code_a) const -{ - return code.value () == code_a.value (); -} -#endif - /** Call the function iff the current error is zero */ nano::error & nano::error::then (std::function next) { @@ -495,14 +432,3 @@ nano::error & nano::error::clear () message.clear (); return *this; } - -#if defined(NANO_USE_BOOST_TO_STD_ERROR_BRIDGE) -// TODO: theoretically, nothing besides template (partial) specializations should ever be added inside std... -namespace std -{ -std::error_code make_error_code (boost::system::errc::errc_t const & e) -{ - return std::error_code (static_cast (e), ::nano::error_conversion::generic_category ()); -} -} -#endif diff --git a/nano/lib/errors.hpp b/nano/lib/errors.hpp index 27e8c6c5f5..d06b5d02b9 100644 --- a/nano/lib/errors.hpp +++ b/nano/lib/errors.hpp @@ -1,8 +1,5 @@ #pragma once -#include -#include - #include #include #include @@ -197,51 +194,6 @@ REGISTER_ERROR_CODES (nano, error_rpc); REGISTER_ERROR_CODES (nano, error_process); REGISTER_ERROR_CODES (nano, error_config); -#if BOOST_VERSION >= 107800 -/* no need for error_code bridge */ -#else -#define NANO_USE_BOOST_TO_STD_ERROR_BRIDGE -#endif - -#if defined(NANO_USE_BOOST_TO_STD_ERROR_BRIDGE) -/* boost->std error_code bridge */ -namespace nano -{ -namespace error_conversion -{ - std::error_category const & generic_category (); -} -} - -namespace std -{ -template <> -struct is_error_code_enum - : public std::true_type -{ -}; - -std::error_code make_error_code (boost::system::errc::errc_t const & e); -} -namespace nano -{ -namespace error_conversion -{ - namespace detail - { - class generic_category : public std::error_category - { - public: - char const * name () const noexcept override; - std::string message (int value) const override; - }; - } - std::error_category const & generic_category (); - std::error_code convert (boost::system::error_code const & error); -} -} -#endif - namespace nano { /** Adapter for std/boost::error_code, std::exception and bool flags to facilitate unified error handling */ @@ -253,24 +205,14 @@ class error error (nano::error && error_a) = default; error (std::error_code code_a); -#if defined(NANO_USE_BOOST_TO_STD_ERROR_BRIDGE) - error (boost::system::error_code const & code_a); -#endif error (std::string message_a); error (std::exception const & exception_a); error & operator= (nano::error const & err_a); error & operator= (nano::error && err_a); error & operator= (std::error_code code_a); -#if defined(NANO_USE_BOOST_TO_STD_ERROR_BRIDGE) - error & operator= (boost::system::error_code const & code_a); - error & operator= (boost::system::errc::errc_t const & code_a); -#endif error & operator= (std::string message_a); error & operator= (std::exception const & exception_a); bool operator== (std::error_code code_a) const; -#if defined(NANO_USE_BOOST_TO_STD_ERROR_BRIDGE) - bool operator== (boost::system::error_code code_a) const; -#endif error & then (std::function next); template error & accept (ErrorCode... err) diff --git a/nano/lib/plat/darwin/thread_role.cpp b/nano/lib/plat/darwin/thread_role.cpp index dd9bdc6199..4726e8f51d 100644 --- a/nano/lib/plat/darwin/thread_role.cpp +++ b/nano/lib/plat/darwin/thread_role.cpp @@ -1,4 +1,4 @@ -#include +#include #include diff --git a/nano/lib/plat/linux/thread_role.cpp b/nano/lib/plat/linux/thread_role.cpp index 46d2952250..d95a158d78 100644 --- a/nano/lib/plat/linux/thread_role.cpp +++ b/nano/lib/plat/linux/thread_role.cpp @@ -1,4 +1,4 @@ -#include +#include #include diff --git a/nano/lib/plat/windows/thread_role.cpp b/nano/lib/plat/windows/thread_role.cpp index a604e67baa..0a02f9f3f6 100644 --- a/nano/lib/plat/windows/thread_role.cpp +++ b/nano/lib/plat/windows/thread_role.cpp @@ -1,6 +1,6 @@ -#include +#include -#include +#include void nano::thread_role::set_os_name (std::string const & thread_name) { diff --git a/nano/lib/processing_queue.hpp b/nano/lib/processing_queue.hpp index 4fa89584e3..e9f317373e 100644 --- a/nano/lib/processing_queue.hpp +++ b/nano/lib/processing_queue.hpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -189,4 +190,4 @@ class processing_queue final nano::condition_variable condition; std::vector threads; }; -} \ No newline at end of file +} diff --git a/nano/lib/rep_weights.cpp b/nano/lib/rep_weights.cpp index 17968e7863..65a0604b4d 100644 --- a/nano/lib/rep_weights.cpp +++ b/nano/lib/rep_weights.cpp @@ -1,5 +1,5 @@ #include -#include +#include void nano::rep_weights::representation_add (nano::account const & source_rep_a, nano::uint128_t const & amount_a) { diff --git a/nano/lib/rep_weights.hpp b/nano/lib/rep_weights.hpp index 4647dd2a97..e7716ee3b2 100644 --- a/nano/lib/rep_weights.hpp +++ b/nano/lib/rep_weights.hpp @@ -9,8 +9,10 @@ namespace nano { -class store; -class transaction; +namespace store +{ + class component; +} class rep_weights { diff --git a/nano/lib/thread_pool.cpp b/nano/lib/thread_pool.cpp new file mode 100644 index 0000000000..571a840718 --- /dev/null +++ b/nano/lib/thread_pool.cpp @@ -0,0 +1,97 @@ +#include + +#include +#include +#include + +/* + * thread_pool + */ + +nano::thread_pool::thread_pool (unsigned num_threads, nano::thread_role::name thread_name) : + num_threads (num_threads), + thread_pool_m (std::make_unique (num_threads)), + thread_names_latch{ num_threads } +{ + set_thread_names (thread_name); +} + +nano::thread_pool::~thread_pool () +{ + stop (); +} + +void nano::thread_pool::stop () +{ + nano::unique_lock lk (mutex); + if (!stopped) + { + stopped = true; +#if defined(BOOST_ASIO_HAS_IOCP) + // A hack needed for Windows to prevent deadlock during destruction, described here: https://github.com/chriskohlhoff/asio/issues/431 + boost::asio::use_service (*thread_pool_m).stop (); +#endif + lk.unlock (); + thread_pool_m->stop (); + thread_pool_m->join (); + lk.lock (); + thread_pool_m = nullptr; + } +} + +void nano::thread_pool::push_task (std::function task) +{ + ++num_tasks; + nano::lock_guard guard (mutex); + if (!stopped) + { + boost::asio::post (*thread_pool_m, [this, task] () { + task (); + --num_tasks; + }); + } +} + +void nano::thread_pool::add_timed_task (std::chrono::steady_clock::time_point const & expiry_time, std::function task) +{ + nano::lock_guard guard (mutex); + if (!stopped && thread_pool_m) + { + auto timer = std::make_shared (thread_pool_m->get_executor (), expiry_time); + timer->async_wait ([this, task, timer] (boost::system::error_code const & ec) { + if (!ec) + { + push_task (task); + } + }); + } +} + +unsigned nano::thread_pool::get_num_threads () const +{ + return num_threads; +} + +uint64_t nano::thread_pool::num_queued_tasks () const +{ + return num_tasks; +} + +void nano::thread_pool::set_thread_names (nano::thread_role::name thread_name) +{ + for (auto i = 0u; i < num_threads; ++i) + { + boost::asio::post (*thread_pool_m, [this, thread_name] () { + nano::thread_role::set (thread_name); + thread_names_latch.arrive_and_wait (); + }); + } + thread_names_latch.wait (); +} + +std::unique_ptr nano::collect_container_info (thread_pool & thread_pool, std::string const & name) +{ + auto composite = std::make_unique (name); + composite->add_component (std::make_unique (container_info{ "count", thread_pool.num_queued_tasks (), sizeof (std::function) })); + return composite; +} diff --git a/nano/lib/thread_pool.hpp b/nano/lib/thread_pool.hpp new file mode 100644 index 0000000000..f56a6271e0 --- /dev/null +++ b/nano/lib/thread_pool.hpp @@ -0,0 +1,53 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include + +namespace boost::asio +{ +class thread_pool; +} + +namespace nano +{ +class thread_pool final +{ +public: + explicit thread_pool (unsigned, nano::thread_role::name); + ~thread_pool (); + + /** This will run when there is an available thread for execution */ + void push_task (std::function); + + /** Run a task at a certain point in time */ + void add_timed_task (std::chrono::steady_clock::time_point const & expiry_time, std::function task); + + /** Stops any further pushed tasks from executing */ + void stop (); + + /** Number of threads in the thread pool */ + unsigned get_num_threads () const; + + /** Returns the number of tasks which are awaiting execution by the thread pool **/ + uint64_t num_queued_tasks () const; + +private: + nano::mutex mutex; + std::atomic stopped{ false }; + unsigned num_threads; + std::unique_ptr thread_pool_m; + nano::relaxed_atomic_integral num_tasks{ 0 }; + + /** Set the names of all the threads in the thread pool for easier identification */ + std::latch thread_names_latch; + void set_thread_names (nano::thread_role::name thread_name); +}; + +std::unique_ptr collect_container_info (thread_pool & thread_pool, std::string const & name); +} // namespace nano diff --git a/nano/lib/thread_runner.cpp b/nano/lib/thread_runner.cpp new file mode 100644 index 0000000000..8e2c308718 --- /dev/null +++ b/nano/lib/thread_runner.cpp @@ -0,0 +1,82 @@ +#include + +#include + +/* + * thread_runner + */ + +nano::thread_runner::thread_runner (boost::asio::io_context & io_ctx_a, unsigned num_threads, const nano::thread_role::name thread_role_a) : + io_guard{ boost::asio::make_work_guard (io_ctx_a) }, + role{ thread_role_a } +{ + for (auto i (0u); i < num_threads; ++i) + { + threads.emplace_back (nano::thread_attributes::get_default (), [this, &io_ctx_a] () { + nano::thread_role::set (role); + try + { + run (io_ctx_a); + } + catch (std::exception const & ex) + { + std::cerr << ex.what () << std::endl; +#ifndef NDEBUG + throw; // Re-throw to debugger in debug mode +#endif + } + catch (...) + { +#ifndef NDEBUG + throw; // Re-throw to debugger in debug mode +#endif + } + }); + } +} + +nano::thread_runner::~thread_runner () +{ + join (); +} + +void nano::thread_runner::run (boost::asio::io_context & io_ctx_a) +{ +#if NANO_ASIO_HANDLER_TRACKING == 0 + io_ctx_a.run (); +#else + nano::timer<> timer; + timer.start (); + while (true) + { + timer.restart (); + // Run at most 1 completion handler and record the time it took to complete (non-blocking) + auto count = io_ctx_a.poll_one (); + if (count == 1 && timer.since_start ().count () >= NANO_ASIO_HANDLER_TRACKING) + { + auto timestamp = std::chrono::duration_cast (std::chrono::system_clock::now ().time_since_epoch ()).count (); + std::cout << (boost::format ("[%1%] io_thread held for %2%ms") % timestamp % timer.since_start ().count ()).str () << std::endl; + } + // Sleep for a bit to give more time slices to other threads + std::this_thread::sleep_for (std::chrono::milliseconds (5)); + std::this_thread::yield (); + } +#endif +} + +void nano::thread_runner::join () +{ + io_guard.reset (); + for (auto & i : threads) + { + if (i.joinable ()) + { + i.join (); + } + } +} + +void nano::thread_runner::stop_event_processing () +{ + io_guard.get_executor ().context ().stop (); +} diff --git a/nano/lib/thread_runner.hpp b/nano/lib/thread_runner.hpp new file mode 100644 index 0000000000..7fbba2e082 --- /dev/null +++ b/nano/lib/thread_runner.hpp @@ -0,0 +1,31 @@ +#pragma once +#include +#include +#include +#include +#include + +#include + +namespace nano +{ +class thread_runner final +{ +public: + thread_runner (boost::asio::io_context &, unsigned num_threads, nano::thread_role::name thread_role = nano::thread_role::name::io); + ~thread_runner (); + + /** Tells the IO context to stop processing events.*/ + void stop_event_processing (); + /** Wait for IO threads to complete */ + void join (); + +private: + nano::thread_role::name const role; + std::vector threads; + boost::asio::executor_work_guard io_guard; + +private: + void run (boost::asio::io_context &); +}; +} // namespace nano diff --git a/nano/lib/threading.cpp b/nano/lib/threading.cpp index d6ef17e71d..6da0e000af 100644 --- a/nano/lib/threading.cpp +++ b/nano/lib/threading.cpp @@ -1,11 +1,7 @@ -#include #include +#include #include -#include - -#include -#include #include /* @@ -19,179 +15,6 @@ boost::thread::attributes nano::thread_attributes::get_default () return attrs; } -/* - * thread_runner - */ - -nano::thread_runner::thread_runner (boost::asio::io_context & io_ctx_a, unsigned num_threads, const nano::thread_role::name thread_role_a) : - io_guard{ boost::asio::make_work_guard (io_ctx_a) }, - role{ thread_role_a } -{ - for (auto i (0u); i < num_threads; ++i) - { - threads.emplace_back (nano::thread_attributes::get_default (), [this, &io_ctx_a] () { - nano::thread_role::set (role); - - // In a release build, catch and swallow any exceptions, - // In debug mode let if fall through - -#ifndef NDEBUG - run (io_ctx_a); -#else - try - { - run (io_ctx_a); - } - catch (std::exception const & ex) - { - std::cerr << ex.what () << std::endl; - } - catch (...) - { - } -#endif - }); - } -} - -nano::thread_runner::~thread_runner () -{ - join (); -} - -void nano::thread_runner::run (boost::asio::io_context & io_ctx_a) -{ -#if NANO_ASIO_HANDLER_TRACKING == 0 - io_ctx_a.run (); -#else - nano::timer<> timer; - timer.start (); - while (true) - { - timer.restart (); - // Run at most 1 completion handler and record the time it took to complete (non-blocking) - auto count = io_ctx_a.poll_one (); - if (count == 1 && timer.since_start ().count () >= NANO_ASIO_HANDLER_TRACKING) - { - auto timestamp = std::chrono::duration_cast (std::chrono::system_clock::now ().time_since_epoch ()).count (); - std::cout << (boost::format ("[%1%] io_thread held for %2%ms") % timestamp % timer.since_start ().count ()).str () << std::endl; - } - // Sleep for a bit to give more time slices to other threads - std::this_thread::sleep_for (std::chrono::milliseconds (5)); - std::this_thread::yield (); - } -#endif -} - -void nano::thread_runner::join () -{ - io_guard.reset (); - for (auto & i : threads) - { - if (i.joinable ()) - { - i.join (); - } - } -} - -void nano::thread_runner::stop_event_processing () -{ - io_guard.get_executor ().context ().stop (); -} - -/* - * thread_pool - */ - -nano::thread_pool::thread_pool (unsigned num_threads, nano::thread_role::name thread_name) : - num_threads (num_threads), - thread_pool_m (std::make_unique (num_threads)), - thread_names_latch{ num_threads } -{ - set_thread_names (thread_name); -} - -nano::thread_pool::~thread_pool () -{ - stop (); -} - -void nano::thread_pool::stop () -{ - nano::unique_lock lk (mutex); - if (!stopped) - { - stopped = true; -#if defined(BOOST_ASIO_HAS_IOCP) - // A hack needed for Windows to prevent deadlock during destruction, described here: https://github.com/chriskohlhoff/asio/issues/431 - boost::asio::use_service (*thread_pool_m).stop (); -#endif - lk.unlock (); - thread_pool_m->stop (); - thread_pool_m->join (); - lk.lock (); - thread_pool_m = nullptr; - } -} - -void nano::thread_pool::push_task (std::function task) -{ - ++num_tasks; - nano::lock_guard guard (mutex); - if (!stopped) - { - boost::asio::post (*thread_pool_m, [this, task] () { - task (); - --num_tasks; - }); - } -} - -void nano::thread_pool::add_timed_task (std::chrono::steady_clock::time_point const & expiry_time, std::function task) -{ - nano::lock_guard guard (mutex); - if (!stopped && thread_pool_m) - { - auto timer = std::make_shared (thread_pool_m->get_executor (), expiry_time); - timer->async_wait ([this, task, timer] (boost::system::error_code const & ec) { - if (!ec) - { - push_task (task); - } - }); - } -} - -unsigned nano::thread_pool::get_num_threads () const -{ - return num_threads; -} - -uint64_t nano::thread_pool::num_queued_tasks () const -{ - return num_tasks; -} - -void nano::thread_pool::set_thread_names (nano::thread_role::name thread_name) -{ - for (auto i = 0u; i < num_threads; ++i) - { - boost::asio::post (*thread_pool_m, [this, thread_name] () { - nano::thread_role::set (thread_name); - thread_names_latch.arrive_and_wait (); - }); - } - thread_names_latch.wait (); -} - -std::unique_ptr nano::collect_container_info (thread_pool & thread_pool, std::string const & name) -{ - auto composite = std::make_unique (name); - composite->add_component (std::make_unique (container_info{ "count", thread_pool.num_queued_tasks (), sizeof (std::function) })); - return composite; -} - unsigned int nano::hardware_concurrency () { // Try to read overridden value from environment variable diff --git a/nano/lib/threading.hpp b/nano/lib/threading.hpp index 3b82a2f856..17b74a4cb2 100644 --- a/nano/lib/threading.hpp +++ b/nano/lib/threading.hpp @@ -1,80 +1,15 @@ #pragma once -#include -#include -#include -#include -#include -#include -#include #include #include -#include -#include - namespace nano { namespace thread_attributes { boost::thread::attributes get_default (); -} - -class thread_runner final -{ -public: - thread_runner (boost::asio::io_context &, unsigned num_threads, nano::thread_role::name thread_role = nano::thread_role::name::io); - ~thread_runner (); - - /** Tells the IO context to stop processing events.*/ - void stop_event_processing (); - /** Wait for IO threads to complete */ - void join (); - -private: - nano::thread_role::name const role; - std::vector threads; - boost::asio::executor_work_guard io_guard; - -private: - void run (boost::asio::io_context &); -}; - -class thread_pool final -{ -public: - explicit thread_pool (unsigned, nano::thread_role::name); - ~thread_pool (); - - /** This will run when there is an available thread for execution */ - void push_task (std::function); - - /** Run a task at a certain point in time */ - void add_timed_task (std::chrono::steady_clock::time_point const & expiry_time, std::function task); - - /** Stops any further pushed tasks from executing */ - void stop (); - - /** Number of threads in the thread pool */ - unsigned get_num_threads () const; - - /** Returns the number of tasks which are awaiting execution by the thread pool **/ - uint64_t num_queued_tasks () const; - -private: - nano::mutex mutex; - std::atomic stopped{ false }; - unsigned num_threads; - std::unique_ptr thread_pool_m; - nano::relaxed_atomic_integral num_tasks{ 0 }; - - /** Set the names of all the threads in the thread pool for easier identification */ - std::latch thread_names_latch; - void set_thread_names (nano::thread_role::name thread_name); -}; - -std::unique_ptr collect_container_info (thread_pool & thread_pool, std::string const & name); +} // namespace thread_attributes /** * Number of available logical processor cores. Might be overridden by setting `NANO_HARDWARE_CONCURRENCY` environment variable @@ -83,6 +18,7 @@ unsigned int hardware_concurrency (); /** * If thread is joinable joins it, otherwise does nothing + * Returns thread.joinable() */ bool join_or_pass (std::thread &); -} +} // namespace nano diff --git a/nano/lib/tlsconfig.cpp b/nano/lib/tlsconfig.cpp index d16efcfa31..34aade654d 100644 --- a/nano/lib/tlsconfig.cpp +++ b/nano/lib/tlsconfig.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include diff --git a/nano/lib/work.cpp b/nano/lib/work.cpp index 7e02981af2..07b27e3979 100644 --- a/nano/lib/work.cpp +++ b/nano/lib/work.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include diff --git a/nano/load_test/entry.cpp b/nano/load_test/entry.cpp index a2a37123b6..fcd067961d 100644 --- a/nano/load_test/entry.cpp +++ b/nano/load_test/entry.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include diff --git a/nano/nano_node/daemon.cpp b/nano/nano_node/daemon.cpp index a8431fc68a..50b4afdafd 100644 --- a/nano/nano_node/daemon.cpp +++ b/nano/nano_node/daemon.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include diff --git a/nano/nano_node/entry.cpp b/nano/nano_node/entry.cpp index e859d2b437..ab21ab03c5 100644 --- a/nano/nano_node/entry.cpp +++ b/nano/nano_node/entry.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -8,6 +9,7 @@ #include #include #include +#include #include #include @@ -306,7 +308,7 @@ int main (int argc, char * const * argv) for (; i != end; ++i) { nano::block_hash hash = i->first; - nano::block_w_sideband sideband = i->second; + nano::store::block_w_sideband sideband = i->second; std::shared_ptr b = sideband.block; std::cout << hash.to_string () << std::endl << b->to_json (); @@ -1406,7 +1408,7 @@ int main (int argc, char * const * argv) } }; - auto check_account = [&print_error_message, &silent, &count, &block_count] (std::shared_ptr const & node, nano::read_transaction const & transaction, nano::account const & account, nano::account_info const & info) { + auto check_account = [&print_error_message, &silent, &count, &block_count] (std::shared_ptr const & node, nano::store::read_transaction const & transaction, nano::account const & account, nano::account_info const & info) { ++count; if (!silent && (count % 20000) == 0) { @@ -1570,7 +1572,7 @@ int main (int argc, char * const * argv) // Check link epoch version if (sideband.details.is_receive && (!node->ledger.pruning || !node->store.pruned.exists (transaction, block->link ().as_block_hash ()))) { - if (sideband.source_epoch != node->store.block.version (transaction, block->link ().as_block_hash ())) + if (sideband.source_epoch != node->ledger.version (*block)) { print_error_message (boost::str (boost::format ("Incorrect source epoch for block %1%\n") % hash.to_string ())); } @@ -1680,14 +1682,14 @@ int main (int argc, char * const * argv) finished = false; std::deque> pending; - auto check_pending = [&print_error_message, &silent, &count] (std::shared_ptr const & node, nano::read_transaction const & transaction, nano::pending_key const & key, nano::pending_info const & info) { + auto check_pending = [&print_error_message, &silent, &count] (std::shared_ptr const & node, nano::store::read_transaction const & transaction, nano::pending_key const & key, nano::pending_info const & info) { ++count; if (!silent && (count % 500000) == 0) { std::cout << boost::str (boost::format ("%1% pending blocks validated\n") % count); } // Check block existance - auto block (node->store.block.get_no_sideband (transaction, key.hash)); + auto block (node->store.block.get (transaction, key.hash)); bool pruned (false); if (block == nullptr) { @@ -1811,7 +1813,7 @@ int main (int argc, char * const * argv) while (!hash.is_zero ()) { // Retrieving block data - auto block (source_node->store.block.get_no_sideband (transaction, hash)); + auto block (source_node->store.block.get (transaction, hash)); if (block != nullptr) { ++count; @@ -1914,7 +1916,7 @@ int main (int argc, char * const * argv) nano::locked>> opened_account_versions_shared (epoch_count); using opened_account_versions_t = decltype (opened_account_versions_shared)::value_type; node->store.account.for_each_par ( - [&opened_account_versions_shared, epoch_count] (nano::read_transaction const & /*unused*/, nano::store_iterator i, nano::store_iterator n) { + [&opened_account_versions_shared, epoch_count] (nano::store::read_transaction const & /*unused*/, nano::store::iterator i, nano::store::iterator n) { // First cache locally opened_account_versions_t opened_account_versions_l (epoch_count); for (; i != n; ++i) @@ -1951,7 +1953,7 @@ int main (int argc, char * const * argv) nano::locked>> unopened_highest_pending_shared; using unopened_highest_pending_t = decltype (unopened_highest_pending_shared)::value_type; node->store.pending.for_each_par ( - [&unopened_highest_pending_shared, &opened_accounts] (nano::read_transaction const & /*unused*/, nano::store_iterator i, nano::store_iterator n) { + [&unopened_highest_pending_shared, &opened_accounts] (nano::store::read_transaction const & /*unused*/, nano::store::iterator i, nano::store::iterator n) { // First cache locally unopened_highest_pending_t unopened_highest_pending_l; for (; i != n; ++i) diff --git a/nano/nano_rpc/entry.cpp b/nano/nano_rpc/entry.cpp index 49f5bc88a2..0dd7b79d86 100644 --- a/nano/nano_rpc/entry.cpp +++ b/nano/nano_rpc/entry.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include diff --git a/nano/nano_wallet/entry.cpp b/nano/nano_wallet/entry.cpp index 39681540c5..906841cb79 100644 --- a/nano/nano_wallet/entry.cpp +++ b/nano/nano_wallet/entry.cpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/nano/node/CMakeLists.txt b/nano/node/CMakeLists.txt index 4e1f29cc02..01afab48f9 100644 --- a/nano/node/CMakeLists.txt +++ b/nano/node/CMakeLists.txt @@ -112,37 +112,10 @@ add_library( json_handler.cpp ledger_walker.hpp ledger_walker.cpp - lmdb/account_store.hpp - lmdb/account_store.cpp - lmdb/block_store.hpp - lmdb/block_store.cpp - lmdb/confirmation_height_store.hpp - lmdb/confirmation_height_store.cpp - lmdb/final_vote_store.hpp - lmdb/final_vote_store.cpp - lmdb/frontier_store.hpp - lmdb/frontier_store.cpp - lmdb/online_weight_store.hpp - lmdb/online_weight_store.cpp - lmdb/peer_store.hpp - lmdb/peer_store.cpp - lmdb/pending_store.hpp - lmdb/pending_store.cpp - lmdb/pruned_store.hpp - lmdb/pruned_store.cpp - lmdb/version_store.hpp - lmdb/version_store.cpp - lmdb/lmdb.hpp - lmdb/lmdb.cpp - lmdb/lmdb_env.hpp - lmdb/lmdb_env.cpp - lmdb/lmdb_iterator.hpp - lmdb/lmdb_txn.hpp - lmdb/lmdb_txn.cpp - lmdb/wallet_value.hpp - lmdb/wallet_value.cpp logging.hpp logging.cpp + make_store.hpp + make_store.cpp network.hpp network.cpp nodeconfig.hpp @@ -169,31 +142,6 @@ add_library( repcrawler.cpp request_aggregator.hpp request_aggregator.cpp - rocksdb/account_store.hpp - rocksdb/account_store.cpp - rocksdb/block_store.hpp - rocksdb/block_store.cpp - rocksdb/confirmation_height_store.hpp - rocksdb/confirmation_height_store.cpp - rocksdb/final_vote_store.hpp - rocksdb/final_vote_store.cpp - rocksdb/frontier_store.hpp - rocksdb/frontier_store.cpp - rocksdb/online_weight_store.hpp - rocksdb/online_weight_store.cpp - rocksdb/peer_store.hpp - rocksdb/peer_store.cpp - rocksdb/pending_store.hpp - rocksdb/pending_store.cpp - rocksdb/pruned_store.hpp - rocksdb/pruned_store.cpp - rocksdb/version_store.hpp - rocksdb/version_store.cpp - rocksdb/rocksdb.hpp - rocksdb/rocksdb.cpp - rocksdb/rocksdb_iterator.hpp - rocksdb/rocksdb_txn.hpp - rocksdb/rocksdb_txn.cpp scheduler/bucket.cpp scheduler/bucket.hpp scheduler/buckets.cpp @@ -258,6 +206,7 @@ endif() target_link_libraries( node + nano_store rpc secure nano_lib @@ -287,6 +236,5 @@ add_dependencies(node ipc_flatbuffers_lib) include_directories(${CMAKE_SOURCE_DIR}/submodules) include_directories(${CMAKE_SOURCE_DIR}/submodules/cpptoml/include) -include_directories(${CMAKE_SOURCE_DIR}/submodules/lmdb) include_directories( ${CMAKE_SOURCE_DIR}/submodules/nano-pow-server/deps/cpptoml/include) diff --git a/nano/node/active_transactions.cpp b/nano/node/active_transactions.cpp index ca00132a9c..05972cdc66 100644 --- a/nano/node/active_transactions.cpp +++ b/nano/node/active_transactions.cpp @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include @@ -624,7 +624,7 @@ bool nano::active_transactions::publish (std::shared_ptr const & bl } // Returns the type of election status requiring callbacks calling later -boost::optional nano::active_transactions::confirm_block (nano::transaction const & transaction_a, std::shared_ptr const & block_a) +boost::optional nano::active_transactions::confirm_block (store::transaction const & transaction_a, std::shared_ptr const & block_a) { auto const hash = block_a->hash (); std::shared_ptr election = nullptr; diff --git a/nano/node/active_transactions.hpp b/nano/node/active_transactions.hpp index 865da63be0..bf54078db6 100644 --- a/nano/node/active_transactions.hpp +++ b/nano/node/active_transactions.hpp @@ -28,7 +28,6 @@ class block; class block_sideband; class election; class vote; -class transaction; class confirmation_height_processor; class stats; @@ -161,7 +160,7 @@ class active_transactions final bool empty () const; std::size_t size () const; bool publish (std::shared_ptr const &); - boost::optional confirm_block (nano::transaction const &, std::shared_ptr const &); + boost::optional confirm_block (store::transaction const &, std::shared_ptr const &); void block_cemented_callback (std::shared_ptr const &); void block_already_cemented_callback (nano::block_hash const &); diff --git a/nano/node/backlog_population.cpp b/nano/node/backlog_population.cpp index 517eb982ba..0aac0b0392 100644 --- a/nano/node/backlog_population.cpp +++ b/nano/node/backlog_population.cpp @@ -2,9 +2,9 @@ #include #include #include -#include +#include -nano::backlog_population::backlog_population (const config & config_a, nano::store & store_a, nano::stats & stats_a) : +nano::backlog_population::backlog_population (const config & config_a, nano::store::component & store_a, nano::stats & stats_a) : config_m{ config_a }, store{ store_a }, stats{ stats_a } @@ -111,7 +111,7 @@ void nano::backlog_population::populate_backlog (nano::unique_lock } } -void nano::backlog_population::activate (nano::transaction const & transaction, nano::account const & account) +void nano::backlog_population::activate (store::transaction const & transaction, nano::account const & account) { debug_assert (!activate_callback.empty ()); diff --git a/nano/node/backlog_population.hpp b/nano/node/backlog_population.hpp index 27d9fb5f38..842ac98dbc 100644 --- a/nano/node/backlog_population.hpp +++ b/nano/node/backlog_population.hpp @@ -9,10 +9,14 @@ #include #include +namespace nano::store +{ +class component; +class transaction; +} namespace nano { class stats; -class store; class election_scheduler; class backlog_population final @@ -30,7 +34,7 @@ class backlog_population final unsigned frequency; }; - backlog_population (const config &, nano::store &, nano::stats &); + backlog_population (const config &, store::component &, nano::stats &); ~backlog_population (); void start (); @@ -46,11 +50,11 @@ class backlog_population final /** * Callback called for each backlogged account */ - using callback_t = nano::observer_set; + using callback_t = nano::observer_set; callback_t activate_callback; private: // Dependencies - nano::store & store; + nano::store::component & store; nano::stats & stats; config config_m; @@ -60,7 +64,7 @@ class backlog_population final bool predicate () const; void populate_backlog (nano::unique_lock & lock); - void activate (nano::transaction const &, nano::account const &); + void activate (store::transaction const &, nano::account const &); /** This is a manual trigger, the ongoing backlog population does not use this. * It can be triggered even when backlog population (frontiers confirmation) is disabled. */ diff --git a/nano/node/blockprocessor.cpp b/nano/node/blockprocessor.cpp index 20d61894cb..5fc10808dd 100644 --- a/nano/node/blockprocessor.cpp +++ b/nano/node/blockprocessor.cpp @@ -2,7 +2,7 @@ #include #include #include -#include +#include #include @@ -121,7 +121,7 @@ std::optional nano::block_processor::add_blocking (std::sh return result; } -void nano::block_processor::rollback_competitor (nano::write_transaction const & transaction, nano::block const & block) +void nano::block_processor::rollback_competitor (store::write_transaction const & transaction, nano::block const & block) { auto hash = block.hash (); auto successor = node.ledger.successor (transaction, block.qualified_root ()); @@ -314,7 +314,7 @@ auto nano::block_processor::process_batch (nano::unique_lock & lock return processed; } -nano::process_return nano::block_processor::process_one (nano::write_transaction const & transaction_a, std::shared_ptr block, bool const forced_a) +nano::process_return nano::block_processor::process_one (store::write_transaction const & transaction_a, std::shared_ptr block, bool const forced_a) { nano::process_return result; auto hash (block->hash ()); @@ -460,7 +460,7 @@ nano::process_return nano::block_processor::process_one (nano::write_transaction return result; } -void nano::block_processor::queue_unchecked (nano::write_transaction const & transaction_a, nano::hash_or_account const & hash_or_account_a) +void nano::block_processor::queue_unchecked (store::write_transaction const & transaction_a, nano::hash_or_account const & hash_or_account_a) { node.unchecked.trigger (hash_or_account_a); node.gap_cache.erase (hash_or_account_a.hash); diff --git a/nano/node/blockprocessor.hpp b/nano/node/blockprocessor.hpp index 1a8c84c48d..465bc5f6fd 100644 --- a/nano/node/blockprocessor.hpp +++ b/nano/node/blockprocessor.hpp @@ -10,12 +10,14 @@ #include #include +namespace nano::store +{ +class write_transaction; +} + namespace nano { class node; -class read_transaction; -class transaction; -class write_transaction; class write_database_queue; /** @@ -53,9 +55,9 @@ class block_processor final private: // Roll back block in the ledger that conflicts with 'block' - void rollback_competitor (nano::write_transaction const & transaction, nano::block const & block); - nano::process_return process_one (nano::write_transaction const &, std::shared_ptr block, bool const = false); - void queue_unchecked (nano::write_transaction const &, nano::hash_or_account const &); + void rollback_competitor (store::write_transaction const & transaction, nano::block const & block); + nano::process_return process_one (store::write_transaction const &, std::shared_ptr block, bool const = false); + void queue_unchecked (store::write_transaction const &, nano::hash_or_account const &); std::deque process_batch (nano::unique_lock &); void process_verified_state_blocks (std::deque &, std::vector const &, std::vector const &, std::vector const &); void add_impl (std::shared_ptr block); diff --git a/nano/node/bootstrap/bootstrap.cpp b/nano/node/bootstrap/bootstrap.cpp index 67b43f9042..f0d5004d1d 100644 --- a/nano/node/bootstrap/bootstrap.cpp +++ b/nano/node/bootstrap/bootstrap.cpp @@ -166,7 +166,7 @@ bool nano::bootstrap_initiator::in_progress () return !attempts_list.empty (); } -void nano::bootstrap_initiator::block_processed (nano::transaction const & tx, nano::process_return const & result, nano::block const & block) +void nano::bootstrap_initiator::block_processed (store::transaction const & tx, nano::process_return const & result, nano::block const & block) { nano::lock_guard lock{ mutex }; for (auto & i : attempts_list) diff --git a/nano/node/bootstrap/bootstrap.hpp b/nano/node/bootstrap/bootstrap.hpp index 5b6f62ef36..016cf547aa 100644 --- a/nano/node/bootstrap/bootstrap.hpp +++ b/nano/node/bootstrap/bootstrap.hpp @@ -14,6 +14,11 @@ namespace mi = boost::multi_index; +namespace nano::store +{ +class transaction; +} + namespace nano { class node; @@ -101,7 +106,7 @@ class bootstrap_initiator final void notify_listeners (bool); void add_observer (std::function const &); bool in_progress (); - void block_processed (nano::transaction const & tx, nano::process_return const & result, nano::block const & block); + void block_processed (store::transaction const & tx, nano::process_return const & result, nano::block const & block); std::shared_ptr connections; std::shared_ptr new_attempt (); bool has_new_attempts (); diff --git a/nano/node/bootstrap/bootstrap_attempt.cpp b/nano/node/bootstrap/bootstrap_attempt.cpp index 8924da58d7..4656ad5f4f 100644 --- a/nano/node/bootstrap/bootstrap_attempt.cpp +++ b/nano/node/bootstrap/bootstrap_attempt.cpp @@ -135,6 +135,6 @@ bool nano::bootstrap_attempt::process_block (std::shared_ptr const return stop_pull; } -void nano::bootstrap_attempt::block_processed (nano::transaction const & tx, nano::process_return const & result, nano::block const & block) +void nano::bootstrap_attempt::block_processed (store::transaction const & tx, nano::process_return const & result, nano::block const & block) { } diff --git a/nano/node/bootstrap/bootstrap_attempt.hpp b/nano/node/bootstrap/bootstrap_attempt.hpp index fca94ac62d..fa9ed15e61 100644 --- a/nano/node/bootstrap/bootstrap_attempt.hpp +++ b/nano/node/bootstrap/bootstrap_attempt.hpp @@ -5,6 +5,11 @@ #include #include +namespace nano::store +{ +class transaction; +} + namespace nano { class node; @@ -29,7 +34,7 @@ class bootstrap_attempt : public std::enable_shared_from_this char const * mode_text (); virtual bool process_block (std::shared_ptr const &, nano::account const &, uint64_t, nano::bulk_pull::count_t, bool, unsigned); virtual void get_information (boost::property_tree::ptree &) = 0; - virtual void block_processed (nano::transaction const & tx, nano::process_return const & result, nano::block const & block); + virtual void block_processed (store::transaction const & tx, nano::process_return const & result, nano::block const & block); nano::mutex next_log_mutex; std::chrono::steady_clock::time_point next_log{ std::chrono::steady_clock::now () }; std::atomic pulling{ 0 }; diff --git a/nano/node/bootstrap/bootstrap_bulk_pull.cpp b/nano/node/bootstrap/bootstrap_bulk_pull.cpp index 8af9bfe1e6..5c0daa4602 100644 --- a/nano/node/bootstrap/bootstrap_bulk_pull.cpp +++ b/nano/node/bootstrap/bootstrap_bulk_pull.cpp @@ -796,7 +796,7 @@ std::pair, std::unique_ptrstore.tx_begin_read ()); auto stream (node->store.pending.begin (stream_transaction, current_key)); - if (stream == nano::store_iterator (nullptr)) + if (stream == store::iterator (nullptr)) { break; } diff --git a/nano/node/bootstrap/bootstrap_server.cpp b/nano/node/bootstrap/bootstrap_server.cpp index 7e0e674381..5bfc959220 100644 --- a/nano/node/bootstrap/bootstrap_server.cpp +++ b/nano/node/bootstrap/bootstrap_server.cpp @@ -2,10 +2,12 @@ #include #include #include -#include +#include +#include +#include // TODO: Make threads configurable -nano::bootstrap_server::bootstrap_server (nano::store & store_a, nano::ledger & ledger_a, nano::network_constants const & network_constants_a, nano::stats & stats_a) : +nano::bootstrap_server::bootstrap_server (nano::store::component & store_a, nano::ledger & ledger_a, nano::network_constants const & network_constants_a, nano::stats & stats_a) : store{ store_a }, ledger{ ledger_a }, network_constants{ network_constants_a }, @@ -150,12 +152,12 @@ void nano::bootstrap_server::process_batch (std::deque & batch) } } -nano::asc_pull_ack nano::bootstrap_server::process (nano::transaction const & transaction, const nano::asc_pull_req & message) +nano::asc_pull_ack nano::bootstrap_server::process (store::transaction const & transaction, const nano::asc_pull_req & message) { return std::visit ([this, &transaction, &message] (auto && request) { return process (transaction, message.id, request); }, message.payload); } -nano::asc_pull_ack nano::bootstrap_server::process (const nano::transaction &, nano::asc_pull_req::id_t id, const nano::empty_payload & request) +nano::asc_pull_ack nano::bootstrap_server::process (const store::transaction &, nano::asc_pull_req::id_t id, const nano::empty_payload & request) { // Empty payload should never be possible, but return empty response anyway debug_assert (false, "missing payload"); @@ -169,7 +171,7 @@ nano::asc_pull_ack nano::bootstrap_server::process (const nano::transaction &, n * Blocks response */ -nano::asc_pull_ack nano::bootstrap_server::process (nano::transaction const & transaction, nano::asc_pull_req::id_t id, nano::asc_pull_req::blocks_payload const & request) +nano::asc_pull_ack nano::bootstrap_server::process (store::transaction const & transaction, nano::asc_pull_req::id_t id, nano::asc_pull_req::blocks_payload const & request) { const std::size_t count = std::min (static_cast (request.count), max_blocks); @@ -199,7 +201,7 @@ nano::asc_pull_ack nano::bootstrap_server::process (nano::transaction const & tr return prepare_empty_blocks_response (id); } -nano::asc_pull_ack nano::bootstrap_server::prepare_response (nano::transaction const & transaction, nano::asc_pull_req::id_t id, nano::block_hash start_block, std::size_t count) +nano::asc_pull_ack nano::bootstrap_server::prepare_response (store::transaction const & transaction, nano::asc_pull_req::id_t id, nano::block_hash start_block, std::size_t count) { debug_assert (count <= max_blocks); @@ -231,7 +233,7 @@ nano::asc_pull_ack nano::bootstrap_server::prepare_empty_blocks_response (nano:: return response; } -std::vector> nano::bootstrap_server::prepare_blocks (nano::transaction const & transaction, nano::block_hash start_block, std::size_t count) const +std::vector> nano::bootstrap_server::prepare_blocks (store::transaction const & transaction, nano::block_hash start_block, std::size_t count) const { debug_assert (count <= max_blocks); @@ -254,7 +256,7 @@ std::vector> nano::bootstrap_server::prepare_blocks * Account info response */ -nano::asc_pull_ack nano::bootstrap_server::process (const nano::transaction & transaction, nano::asc_pull_req::id_t id, const nano::asc_pull_req::account_info_payload & request) +nano::asc_pull_ack nano::bootstrap_server::process (const store::transaction & transaction, nano::asc_pull_req::id_t id, const nano::asc_pull_req::account_info_payload & request) { nano::asc_pull_ack response{ network_constants }; response.id = id; diff --git a/nano/node/bootstrap/bootstrap_server.hpp b/nano/node/bootstrap/bootstrap_server.hpp index b614661ada..98fd3823f9 100644 --- a/nano/node/bootstrap/bootstrap_server.hpp +++ b/nano/node/bootstrap/bootstrap_server.hpp @@ -7,6 +7,11 @@ #include #include +namespace nano::store +{ +class transaction; +} + namespace nano { class ledger; @@ -29,7 +34,7 @@ class bootstrap_server final using request_t = std::pair>; // public: - bootstrap_server (nano::store &, nano::ledger &, nano::network_constants const &, nano::stats &); + bootstrap_server (nano::store::component &, nano::ledger &, nano::network_constants const &, nano::stats &); ~bootstrap_server (); void start (); @@ -46,23 +51,23 @@ class bootstrap_server final private: void process_batch (std::deque & batch); - nano::asc_pull_ack process (nano::transaction const &, nano::asc_pull_req const & message); + nano::asc_pull_ack process (store::transaction const &, nano::asc_pull_req const & message); void respond (nano::asc_pull_ack &, std::shared_ptr &); - nano::asc_pull_ack process (nano::transaction const &, nano::asc_pull_req::id_t id, nano::empty_payload const & request); + nano::asc_pull_ack process (store::transaction const &, nano::asc_pull_req::id_t id, nano::empty_payload const & request); /* * Blocks response */ - nano::asc_pull_ack process (nano::transaction const &, nano::asc_pull_req::id_t id, nano::asc_pull_req::blocks_payload const & request); - nano::asc_pull_ack prepare_response (nano::transaction const &, nano::asc_pull_req::id_t id, nano::block_hash start_block, std::size_t count); + nano::asc_pull_ack process (store::transaction const &, nano::asc_pull_req::id_t id, nano::asc_pull_req::blocks_payload const & request); + nano::asc_pull_ack prepare_response (store::transaction const &, nano::asc_pull_req::id_t id, nano::block_hash start_block, std::size_t count); nano::asc_pull_ack prepare_empty_blocks_response (nano::asc_pull_req::id_t id); - std::vector> prepare_blocks (nano::transaction const &, nano::block_hash start_block, std::size_t count) const; + std::vector> prepare_blocks (store::transaction const &, nano::block_hash start_block, std::size_t count) const; /* * Account info response */ - nano::asc_pull_ack process (nano::transaction const &, nano::asc_pull_req::id_t id, nano::asc_pull_req::account_info_payload const & request); + nano::asc_pull_ack process (store::transaction const &, nano::asc_pull_req::id_t id, nano::asc_pull_req::account_info_payload const & request); /* * Checks if the request should be dropped early on @@ -71,7 +76,7 @@ class bootstrap_server final bool verify_request_type (nano::asc_pull_type) const; private: // Dependencies - nano::store & store; + nano::store::component & store; nano::ledger & ledger; nano::network_constants const & network_constants; nano::stats & stats; @@ -83,4 +88,4 @@ class bootstrap_server final /** Maximum number of blocks to send in a single response, cannot be higher than capacity of a single `asc_pull_ack` message */ constexpr static std::size_t max_blocks = nano::asc_pull_ack::blocks_payload::max_blocks; }; -} \ No newline at end of file +} diff --git a/nano/node/bootstrap_ascending/iterators.cpp b/nano/node/bootstrap_ascending/iterators.cpp index 1ba41b9dd8..d35c682567 100644 --- a/nano/node/bootstrap_ascending/iterators.cpp +++ b/nano/node/bootstrap_ascending/iterators.cpp @@ -1,13 +1,15 @@ #include #include #include -#include +#include +#include +#include /* * database_iterator */ -nano::bootstrap_ascending::database_iterator::database_iterator (nano::store & store_a, table_type table_a) : +nano::bootstrap_ascending::database_iterator::database_iterator (nano::store::component & store_a, table_type table_a) : store{ store_a }, table{ table_a } { @@ -18,7 +20,7 @@ nano::account nano::bootstrap_ascending::database_iterator::operator* () const return current; } -void nano::bootstrap_ascending::database_iterator::next (nano::transaction & tx) +void nano::bootstrap_ascending::database_iterator::next (store::transaction & tx) { switch (table) { @@ -57,7 +59,7 @@ void nano::bootstrap_ascending::database_iterator::next (nano::transaction & tx) * buffered_iterator */ -nano::bootstrap_ascending::buffered_iterator::buffered_iterator (nano::store & store_a) : +nano::bootstrap_ascending::buffered_iterator::buffered_iterator (nano::store::component & store_a) : store{ store_a }, accounts_iterator{ store, database_iterator::table_type::account }, pending_iterator{ store, database_iterator::table_type::pending } diff --git a/nano/node/bootstrap_ascending/iterators.hpp b/nano/node/bootstrap_ascending/iterators.hpp index 82285ecbd0..58d7c6c690 100644 --- a/nano/node/bootstrap_ascending/iterators.hpp +++ b/nano/node/bootstrap_ascending/iterators.hpp @@ -4,53 +4,53 @@ #include -namespace nano +namespace nano::store { -class store; +class component; class transaction; +} -namespace bootstrap_ascending +namespace nano::bootstrap_ascending { - class database_iterator +class database_iterator +{ +public: + enum class table_type { - public: - enum class table_type - { - account, - pending - }; - - explicit database_iterator (nano::store & store, table_type); - nano::account operator* () const; - void next (nano::transaction & tx); - - private: - nano::store & store; - nano::account current{ 0 }; - const table_type table; + account, + pending }; - class buffered_iterator - { - public: - explicit buffered_iterator (nano::store & store); - nano::account operator* () const; - nano::account next (); - // Indicates if a full ledger iteration has taken place e.g. warmed up - bool warmup () const; - - private: - void fill (); + explicit database_iterator (nano::store::component & store, table_type); + nano::account operator* () const; + void next (nano::store::transaction & tx); - private: - nano::store & store; - std::deque buffer; - bool warmup_m{ true }; +private: + nano::store::component & store; + nano::account current{ 0 }; + const table_type table; +}; - database_iterator accounts_iterator; - database_iterator pending_iterator; - - static std::size_t constexpr size = 1024; - }; -} // nano -} // bootstrap_ascending +class buffered_iterator +{ +public: + explicit buffered_iterator (nano::store::component & store); + nano::account operator* () const; + nano::account next (); + // Indicates if a full ledger iteration has taken place e.g. warmed up + bool warmup () const; + +private: + void fill (); + +private: + nano::store::component & store; + std::deque buffer; + bool warmup_m{ true }; + + database_iterator accounts_iterator; + database_iterator pending_iterator; + + static std::size_t constexpr size = 1024; +}; +} // nano::bootstrap_ascending diff --git a/nano/node/bootstrap_ascending/service.cpp b/nano/node/bootstrap_ascending/service.cpp index db05849bb0..52b9b317b9 100644 --- a/nano/node/bootstrap_ascending/service.cpp +++ b/nano/node/bootstrap_ascending/service.cpp @@ -6,7 +6,8 @@ #include #include #include -#include +#include +#include using namespace std::chrono_literals; @@ -124,7 +125,7 @@ std::size_t nano::bootstrap_ascending::service::score_size () const - Marks an account as blocked if the result code is gap source as there is no reason request additional blocks for this account until the dependency is resolved - Marks an account as forwarded if it has been recently referenced by a block that has been inserted. */ -void nano::bootstrap_ascending::service::inspect (nano::transaction const & tx, nano::process_return const & result, nano::block const & block) +void nano::bootstrap_ascending::service::inspect (store::transaction const & tx, nano::process_return const & result, nano::block const & block) { auto const hash = block.hash (); diff --git a/nano/node/bootstrap_ascending/service.hpp b/nano/node/bootstrap_ascending/service.hpp index 0da4ee6dee..a518b3b3da 100644 --- a/nano/node/bootstrap_ascending/service.hpp +++ b/nano/node/bootstrap_ascending/service.hpp @@ -27,7 +27,6 @@ class block_processor; class ledger; class network; class node_config; -class transaction; namespace transport { @@ -89,7 +88,7 @@ namespace bootstrap_ascending private: /* Inspects a block that has been processed by the block processor */ - void inspect (nano::transaction const &, nano::process_return const & result, nano::block const & block); + void inspect (store::transaction const &, nano::process_return const & result, nano::block const & block); void throttle_if_needed (nano::unique_lock & lock); void run (); diff --git a/nano/node/cli.cpp b/nano/node/cli.cpp index 5dd2c39f85..b3b63447ed 100644 --- a/nano/node/cli.cpp +++ b/nano/node/cli.cpp @@ -6,11 +6,12 @@ #include #include +#include #include namespace { -void reset_confirmation_heights (nano::write_transaction const & transaction, nano::ledger_constants & constants, nano::store & store); +void reset_confirmation_heights (nano::store::write_transaction const & transaction, nano::ledger_constants & constants, nano::store::component & store); bool is_using_rocksdb (boost::filesystem::path const & data_path, boost::program_options::variables_map const & vm, std::error_code & ec); } @@ -1315,7 +1316,7 @@ std::unique_ptr nano::default_inactive_node (boost::filesys namespace { -void reset_confirmation_heights (nano::write_transaction const & transaction, nano::ledger_constants & constants, nano::store & store) +void reset_confirmation_heights (nano::store::write_transaction const & transaction, nano::ledger_constants & constants, nano::store::component & store) { // First do a clean sweep store.confirmation_height.clear (transaction); diff --git a/nano/node/confirmation_height_bounded.cpp b/nano/node/confirmation_height_bounded.cpp index 7ba0767ab2..a2301f891d 100644 --- a/nano/node/confirmation_height_bounded.cpp +++ b/nano/node/confirmation_height_bounded.cpp @@ -4,6 +4,9 @@ #include #include #include +#include +#include +#include #include @@ -221,7 +224,7 @@ void nano::confirmation_height_bounded::process (std::shared_ptr or debug_assert (checkpoints.empty ()); } -nano::block_hash nano::confirmation_height_bounded::get_least_unconfirmed_hash_from_top_level (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::account const & account_a, nano::confirmation_height_info const & confirmation_height_info_a, uint64_t & block_height_a) +nano::block_hash nano::confirmation_height_bounded::get_least_unconfirmed_hash_from_top_level (store::transaction const & transaction_a, nano::block_hash const & hash_a, nano::account const & account_a, nano::confirmation_height_info const & confirmation_height_info_a, uint64_t & block_height_a) { nano::block_hash least_unconfirmed_hash = hash_a; if (confirmation_height_info_a.height != 0) @@ -245,7 +248,7 @@ nano::block_hash nano::confirmation_height_bounded::get_least_unconfirmed_hash_f return least_unconfirmed_hash; } -bool nano::confirmation_height_bounded::iterate (nano::read_transaction const & transaction_a, uint64_t bottom_height_a, nano::block_hash const & bottom_hash_a, boost::circular_buffer_space_optimized & checkpoints_a, nano::block_hash & top_most_non_receive_block_hash_a, nano::block_hash const & top_level_hash_a, boost::circular_buffer_space_optimized & receive_source_pairs_a, nano::account const & account_a) +bool nano::confirmation_height_bounded::iterate (store::read_transaction const & transaction_a, uint64_t bottom_height_a, nano::block_hash const & bottom_hash_a, boost::circular_buffer_space_optimized & checkpoints_a, nano::block_hash & top_most_non_receive_block_hash_a, nano::block_hash const & top_level_hash_a, boost::circular_buffer_space_optimized & receive_source_pairs_a, nano::account const & account_a) { bool reached_target = false; bool hit_receive = false; @@ -307,7 +310,7 @@ void nano::confirmation_height_bounded::prepare_iterated_blocks_for_cementing (p if (!preparation_data_a.already_cemented) { // Add the non-receive blocks iterated for this account - auto block_height = (ledger.store.block.account_height (preparation_data_a.transaction, preparation_data_a.top_most_non_receive_block_hash)); + auto block_height = (ledger.height (preparation_data_a.transaction, preparation_data_a.top_most_non_receive_block_hash)); if (block_height > preparation_data_a.confirmation_height_info.height) { confirmed_info confirmed_info_l{ block_height, preparation_data_a.top_most_non_receive_block_hash }; diff --git a/nano/node/confirmation_height_bounded.hpp b/nano/node/confirmation_height_bounded.hpp index 338223fba3..c176b41686 100644 --- a/nano/node/confirmation_height_bounded.hpp +++ b/nano/node/confirmation_height_bounded.hpp @@ -1,16 +1,16 @@ #pragma once #include +#include #include #include -#include +#include #include namespace nano { class ledger; -class read_transaction; class logging; class logger_mt; class write_database_queue; @@ -89,7 +89,7 @@ class confirmation_height_bounded final class preparation_data final { public: - nano::transaction const & transaction; + store::transaction const & transaction; nano::block_hash const & top_most_non_receive_block_hash; bool already_cemented; boost::circular_buffer_space_optimized & checkpoints; @@ -114,9 +114,9 @@ class confirmation_height_bounded final nano::timer timer; top_and_next_hash get_next_block (boost::optional const &, boost::circular_buffer_space_optimized const &, boost::circular_buffer_space_optimized const & receive_source_pairs, boost::optional &, nano::block const & original_block); - nano::block_hash get_least_unconfirmed_hash_from_top_level (nano::transaction const &, nano::block_hash const &, nano::account const &, nano::confirmation_height_info const &, uint64_t &); + nano::block_hash get_least_unconfirmed_hash_from_top_level (store::transaction const &, nano::block_hash const &, nano::account const &, nano::confirmation_height_info const &, uint64_t &); void prepare_iterated_blocks_for_cementing (preparation_data &); - bool iterate (nano::read_transaction const &, uint64_t, nano::block_hash const &, boost::circular_buffer_space_optimized &, nano::block_hash &, nano::block_hash const &, boost::circular_buffer_space_optimized &, nano::account const &); + bool iterate (store::read_transaction const &, uint64_t, nano::block_hash const &, boost::circular_buffer_space_optimized &, nano::block_hash &, nano::block_hash const &, boost::circular_buffer_space_optimized &, nano::account const &); nano::ledger & ledger; nano::write_database_queue & write_database_queue; diff --git a/nano/node/confirmation_height_processor.cpp b/nano/node/confirmation_height_processor.cpp index df4be6c490..6cf9772999 100644 --- a/nano/node/confirmation_height_processor.cpp +++ b/nano/node/confirmation_height_processor.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #include #include #include diff --git a/nano/node/confirmation_height_processor.hpp b/nano/node/confirmation_height_processor.hpp index a32b48f381..88f2d1a15b 100644 --- a/nano/node/confirmation_height_processor.hpp +++ b/nano/node/confirmation_height_processor.hpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include diff --git a/nano/node/confirmation_height_unbounded.cpp b/nano/node/confirmation_height_unbounded.cpp index 5d8690c3b5..9a458b9c8c 100644 --- a/nano/node/confirmation_height_unbounded.cpp +++ b/nano/node/confirmation_height_unbounded.cpp @@ -1,8 +1,12 @@ +#include #include #include #include #include #include +#include +#include +#include #include @@ -189,7 +193,7 @@ void nano::confirmation_height_unbounded::process (std::shared_ptr } while ((!receive_source_pairs.empty () || current != original_block->hash ()) && !stopped); } -void nano::confirmation_height_unbounded::collect_unconfirmed_receive_and_sources_for_account (uint64_t block_height_a, uint64_t confirmation_height_a, std::shared_ptr const & block_a, nano::block_hash const & hash_a, nano::account const & account_a, nano::read_transaction const & transaction_a, std::vector & receive_source_pairs_a, std::vector & block_callback_data_a, std::vector & orig_block_callback_data_a, std::shared_ptr original_block) +void nano::confirmation_height_unbounded::collect_unconfirmed_receive_and_sources_for_account (uint64_t block_height_a, uint64_t confirmation_height_a, std::shared_ptr const & block_a, nano::block_hash const & hash_a, nano::account const & account_a, store::read_transaction const & transaction_a, std::vector & receive_source_pairs_a, std::vector & block_callback_data_a, std::vector & orig_block_callback_data_a, std::shared_ptr original_block) { debug_assert (block_a->hash () == hash_a); auto hash (hash_a); @@ -434,7 +438,7 @@ void nano::confirmation_height_unbounded::cement_blocks (nano::write_guard & sco timer.restart (); } -std::shared_ptr nano::confirmation_height_unbounded::get_block_and_sideband (nano::block_hash const & hash_a, nano::transaction const & transaction_a) +std::shared_ptr nano::confirmation_height_unbounded::get_block_and_sideband (nano::block_hash const & hash_a, store::transaction const & transaction_a) { nano::lock_guard guard (block_cache_mutex); auto block_cache_it = block_cache.find (hash_a); diff --git a/nano/node/confirmation_height_unbounded.hpp b/nano/node/confirmation_height_unbounded.hpp index de8f07c088..0af353e69a 100644 --- a/nano/node/confirmation_height_unbounded.hpp +++ b/nano/node/confirmation_height_unbounded.hpp @@ -1,9 +1,10 @@ #pragma once #include +#include #include #include -#include +#include #include #include @@ -11,7 +12,6 @@ namespace nano { class ledger; -class read_transaction; class logging; class logger_mt; class write_database_queue; @@ -65,7 +65,7 @@ class confirmation_height_unbounded final // This allows the load and stores to use relaxed atomic memory ordering. std::unordered_map confirmed_iterated_pairs; nano::relaxed_atomic_integral confirmed_iterated_pairs_size{ 0 }; - std::shared_ptr get_block_and_sideband (nano::block_hash const &, nano::transaction const &); + std::shared_ptr get_block_and_sideband (nano::block_hash const &, store::transaction const &); std::deque pending_writes; nano::relaxed_atomic_integral pending_writes_size{ 0 }; std::unordered_map> implicit_receive_cemented_mapping; @@ -92,7 +92,7 @@ class confirmation_height_unbounded final std::vector const & orig_block_callback_data; }; - void collect_unconfirmed_receive_and_sources_for_account (uint64_t, uint64_t, std::shared_ptr const &, nano::block_hash const &, nano::account const &, nano::read_transaction const &, std::vector &, std::vector &, std::vector &, std::shared_ptr original_block); + void collect_unconfirmed_receive_and_sources_for_account (uint64_t, uint64_t, std::shared_ptr const &, nano::block_hash const &, nano::account const &, store::read_transaction const &, std::vector &, std::vector &, std::vector &, std::shared_ptr original_block); void prepare_iterated_blocks_for_cementing (preparation_data &); nano::ledger & ledger; diff --git a/nano/node/daemonconfig.cpp b/nano/node/daemonconfig.cpp index 3e232646aa..26e2f7fe7c 100644 --- a/nano/node/daemonconfig.cpp +++ b/nano/node/daemonconfig.cpp @@ -3,6 +3,8 @@ #include #include +#include + #include #include diff --git a/nano/node/election.hpp b/nano/node/election.hpp index 1acc6eb921..721503850a 100644 --- a/nano/node/election.hpp +++ b/nano/node/election.hpp @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include diff --git a/nano/node/epoch_upgrader.cpp b/nano/node/epoch_upgrader.cpp index 2064cf1f2f..a731bb4da8 100644 --- a/nano/node/epoch_upgrader.cpp +++ b/nano/node/epoch_upgrader.cpp @@ -2,7 +2,7 @@ #include #include -nano::epoch_upgrader::epoch_upgrader (nano::node & node_a, nano::ledger & ledger_a, nano::store & store_a, nano::network_params & network_params_a, nano::logger_mt & logger_a) : +nano::epoch_upgrader::epoch_upgrader (nano::node & node_a, nano::ledger & ledger_a, nano::store::component & store_a, nano::network_params & network_params_a, nano::logger_mt & logger_a) : node{ node_a }, ledger{ ledger_a }, store{ store_a }, diff --git a/nano/node/epoch_upgrader.hpp b/nano/node/epoch_upgrader.hpp index fb16b4d564..ab0499fb14 100644 --- a/nano/node/epoch_upgrader.hpp +++ b/nano/node/epoch_upgrader.hpp @@ -10,14 +10,17 @@ namespace nano { class node; class ledger; -class store; +namespace store +{ + class component; +} class network_params; class logger_mt; class epoch_upgrader final { public: - epoch_upgrader (nano::node &, nano::ledger &, nano::store &, nano::network_params &, nano::logger_mt &); + epoch_upgrader (nano::node &, nano::ledger &, nano::store::component &, nano::network_params &, nano::logger_mt &); bool start (nano::raw_key const & prv, nano::epoch epoch, uint64_t count_limit, uint64_t threads); void stop (); @@ -25,7 +28,7 @@ class epoch_upgrader final private: // Dependencies nano::node & node; nano::ledger & ledger; - nano::store & store; + nano::store::component & store; nano::network_params & network_params; nano::logger_mt & logger; @@ -35,4 +38,4 @@ class epoch_upgrader final std::atomic stopped{ false }; nano::locked> epoch_upgrading; }; -} \ No newline at end of file +} diff --git a/nano/node/gap_cache.cpp b/nano/node/gap_cache.cpp index 24342787bf..022544e9c7 100644 --- a/nano/node/gap_cache.cpp +++ b/nano/node/gap_cache.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #include diff --git a/nano/node/gap_cache.hpp b/nano/node/gap_cache.hpp index cd26dd2823..118625fd35 100644 --- a/nano/node/gap_cache.hpp +++ b/nano/node/gap_cache.hpp @@ -16,7 +16,6 @@ namespace nano { class node; -class transaction; /** For each gap in account chains, track arrival time and voters */ class gap_information final diff --git a/nano/node/ipc/ipc_access_config.cpp b/nano/node/ipc/ipc_access_config.cpp index 404ebe8049..e7909a2b6c 100644 --- a/nano/node/ipc/ipc_access_config.cpp +++ b/nano/node/ipc/ipc_access_config.cpp @@ -2,6 +2,7 @@ #include #include +#include namespace { diff --git a/nano/node/ipc/ipc_server.cpp b/nano/node/ipc/ipc_server.cpp index 7712871db6..9eaf1e7509 100644 --- a/nano/node/ipc/ipc_server.cpp +++ b/nano/node/ipc/ipc_server.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include diff --git a/nano/node/json_handler.cpp b/nano/node/json_handler.cpp index a62f1ad355..3e6683da74 100644 --- a/nano/node/json_handler.cpp +++ b/nano/node/json_handler.cpp @@ -23,7 +23,7 @@ void construct_json (nano::container_info_component * component, boost::property using ipc_json_handler_no_arg_func_map = std::unordered_map>; ipc_json_handler_no_arg_func_map create_ipc_json_handler_no_arg_func_map (); auto ipc_json_handler_no_arg_funcs = create_ipc_json_handler_no_arg_func_map (); -bool block_confirmed (nano::node & node, nano::transaction & transaction, nano::block_hash const & hash, bool include_active, bool include_only_confirmed); +bool block_confirmed (nano::node & node, nano::store::transaction & transaction, nano::block_hash const & hash, bool include_active, bool include_only_confirmed); char const * epoch_as_string (nano::epoch); } @@ -204,7 +204,7 @@ std::shared_ptr nano::json_handler::wallet_impl () return nullptr; } -bool nano::json_handler::wallet_locked_impl (nano::transaction const & transaction_a, std::shared_ptr const & wallet_a) +bool nano::json_handler::wallet_locked_impl (store::transaction const & transaction_a, std::shared_ptr const & wallet_a) { bool result (false); if (!ec) @@ -218,7 +218,7 @@ bool nano::json_handler::wallet_locked_impl (nano::transaction const & transacti return result; } -bool nano::json_handler::wallet_account_impl (nano::transaction const & transaction_a, std::shared_ptr const & wallet_a, nano::account const & account_a) +bool nano::json_handler::wallet_account_impl (store::transaction const & transaction_a, std::shared_ptr const & wallet_a, nano::account const & account_a) { bool result (false); if (!ec) @@ -257,7 +257,7 @@ nano::account nano::json_handler::account_impl (std::string account_text, std::e return result; } -nano::account_info nano::json_handler::account_info_impl (nano::transaction const & transaction_a, nano::account const & account_a) +nano::account_info nano::json_handler::account_info_impl (store::transaction const & transaction_a, nano::account const & account_a) { nano::account_info result; if (!ec) @@ -402,7 +402,7 @@ uint64_t nano::json_handler::difficulty_ledger (nano::block const & block_a) // Send check if (block_previous != nullptr) { - details.is_send = node.store.block.balance (transaction, previous) > block_a.balance ().number (); + details.is_send = node.ledger.balance (transaction, previous) > block_a.balance ().number (); details_found = true; } // Epoch check @@ -1653,7 +1653,7 @@ void nano::json_handler::block_create () else if (previous_text.is_initialized () && balance_text.is_initialized () && type == "send") { auto transaction (node.store.tx_begin_read ()); - if (node.store.block.exists (transaction, previous) && node.store.block.balance (transaction, previous) != balance.number ()) + if (node.store.block.exists (transaction, previous) && node.ledger.balance (transaction, previous) != balance.number ()) { ec = nano::error_rpc::block_create_balance_mismatch; } @@ -2377,7 +2377,7 @@ namespace class history_visitor : public nano::block_visitor { public: - history_visitor (nano::json_handler & handler_a, bool raw_a, nano::transaction & transaction_a, boost::property_tree::ptree & tree_a, nano::block_hash const & hash_a, std::vector const & accounts_filter_a) : + history_visitor (nano::json_handler & handler_a, bool raw_a, nano::store::transaction & transaction_a, boost::property_tree::ptree & tree_a, nano::block_hash const & hash_a, std::vector const & accounts_filter_a) : handler (handler_a), raw (raw_a), transaction (transaction_a), @@ -2569,7 +2569,7 @@ class history_visitor : public nano::block_visitor } nano::json_handler & handler; bool raw; - nano::transaction & transaction; + nano::store::transaction & transaction; boost::property_tree::ptree & tree; nano::block_hash const & hash; std::vector const & accounts_filter; @@ -5095,7 +5095,7 @@ void nano::json_handler::work_generate () auto transaction_l (node.store.tx_begin_read ()); if (node.store.block.exists (transaction_l, hash)) { - account = node.store.block.account (transaction_l, hash); + account = node.ledger.account (transaction_l, hash); } } auto secondary_work_peers_l (request.get ("secondary_work_peers", false)); @@ -5448,7 +5448,7 @@ ipc_json_handler_no_arg_func_map create_ipc_json_handler_no_arg_func_map () } /** Due to the asynchronous nature of updating confirmation heights, it can also be necessary to check active roots */ -bool block_confirmed (nano::node & node, nano::transaction & transaction, nano::block_hash const & hash, bool include_active, bool include_only_confirmed) +bool block_confirmed (nano::node & node, nano::store::transaction & transaction, nano::block_hash const & hash, bool include_active, bool include_only_confirmed) { bool is_confirmed = false; if (include_active && !include_only_confirmed) diff --git a/nano/node/json_handler.hpp b/nano/node/json_handler.hpp index 60a257e3ab..cb7da62c72 100644 --- a/nano/node/json_handler.hpp +++ b/nano/node/json_handler.hpp @@ -155,10 +155,10 @@ class json_handler : public std::enable_shared_from_this std::string action; boost::property_tree::ptree response_l; std::shared_ptr wallet_impl (); - bool wallet_locked_impl (nano::transaction const &, std::shared_ptr const &); - bool wallet_account_impl (nano::transaction const &, std::shared_ptr const &, nano::account const &); + bool wallet_locked_impl (store::transaction const &, std::shared_ptr const &); + bool wallet_account_impl (store::transaction const &, std::shared_ptr const &, nano::account const &); nano::account account_impl (std::string = "", std::error_code = nano::error_common::bad_account_number); - nano::account_info account_info_impl (nano::transaction const &, nano::account const &); + nano::account_info account_info_impl (store::transaction const &, nano::account const &); nano::amount amount_impl (); std::shared_ptr block_impl (bool = true); nano::block_hash hash_impl (std::string = "hash"); diff --git a/nano/node/ledger_walker.cpp b/nano/node/ledger_walker.cpp index 17d24c1f57..075cd1728e 100644 --- a/nano/node/ledger_walker.cpp +++ b/nano/node/ledger_walker.cpp @@ -5,8 +5,9 @@ #include #include #include -#include #include +#include +#include #include #include @@ -169,7 +170,7 @@ void nano::ledger_walker::clear_queue () decltype (blocks_to_walk){}.swap (blocks_to_walk); } -std::shared_ptr nano::ledger_walker::dequeue_block (nano::transaction const & transaction_a) +std::shared_ptr nano::ledger_walker::dequeue_block (store::transaction const & transaction_a) { auto block = ledger.store.block.get (transaction_a, blocks_to_walk.top ()); blocks_to_walk.pop (); diff --git a/nano/node/ledger_walker.hpp b/nano/node/ledger_walker.hpp index 6458d8acdc..5492767da7 100644 --- a/nano/node/ledger_walker.hpp +++ b/nano/node/ledger_walker.hpp @@ -14,11 +14,15 @@ #include +namespace nano::store +{ +class transaction; +} + namespace nano { class block; class ledger; -class transaction; /** Walks the ledger starting from a start block and applying a depth-first search algorithm */ class ledger_walker final @@ -55,7 +59,7 @@ class ledger_walker final bool add_to_walked_blocks (nano::block_hash const & block_hash_a); bool add_to_walked_blocks_disk (nano::block_hash const & block_hash_a); void clear_queue (); - std::shared_ptr dequeue_block (nano::transaction const & transaction_a); + std::shared_ptr dequeue_block (store::transaction const & transaction_a); friend class ledger_walker_genesis_account_longer_Test; }; diff --git a/nano/node/lmdb/account_store.cpp b/nano/node/lmdb/account_store.cpp deleted file mode 100644 index 53b1749b49..0000000000 --- a/nano/node/lmdb/account_store.cpp +++ /dev/null @@ -1,72 +0,0 @@ -#include -#include -#include - -nano::lmdb::account_store::account_store (nano::lmdb::store & store_a) : - store (store_a){}; - -void nano::lmdb::account_store::put (nano::write_transaction const & transaction, nano::account const & account, nano::account_info const & info) -{ - auto status = store.put (transaction, tables::accounts, account, info); - store.release_assert_success (status); -} - -bool nano::lmdb::account_store::get (nano::transaction const & transaction, nano::account const & account, nano::account_info & info) -{ - nano::mdb_val value; - auto status1 (store.get (transaction, tables::accounts, account, value)); - release_assert (store.success (status1) || store.not_found (status1)); - bool result (true); - if (store.success (status1)) - { - nano::bufferstream stream (reinterpret_cast (value.data ()), value.size ()); - result = info.deserialize (stream); - } - return result; -} - -void nano::lmdb::account_store::del (nano::write_transaction const & transaction_a, nano::account const & account_a) -{ - auto status = store.del (transaction_a, tables::accounts, account_a); - store.release_assert_success (status); -} - -bool nano::lmdb::account_store::exists (nano::transaction const & transaction_a, nano::account const & account_a) -{ - auto iterator (begin (transaction_a, account_a)); - return iterator != end () && nano::account (iterator->first) == account_a; -} - -size_t nano::lmdb::account_store::count (nano::transaction const & transaction_a) -{ - return store.count (transaction_a, tables::accounts); -} - -nano::store_iterator nano::lmdb::account_store::begin (nano::transaction const & transaction, nano::account const & account) const -{ - return store.make_iterator (transaction, tables::accounts, account); -} - -nano::store_iterator nano::lmdb::account_store::begin (nano::transaction const & transaction) const -{ - return store.make_iterator (transaction, tables::accounts); -} - -nano::store_iterator nano::lmdb::account_store::rbegin (nano::transaction const & transaction_a) const -{ - return store.make_iterator (transaction_a, tables::accounts, false); -} - -nano::store_iterator nano::lmdb::account_store::end () const -{ - return nano::store_iterator (nullptr); -} - -void nano::lmdb::account_store::for_each_par (std::function, nano::store_iterator)> const & action_a) const -{ - parallel_traversal ( - [&action_a, this] (nano::uint256_t const & start, nano::uint256_t const & end, bool const is_last) { - auto transaction (this->store.tx_begin_read ()); - action_a (transaction, this->begin (transaction, start), !is_last ? this->begin (transaction, end) : this->end ()); - }); -} diff --git a/nano/node/lmdb/account_store.hpp b/nano/node/lmdb/account_store.hpp deleted file mode 100644 index ea93b257fd..0000000000 --- a/nano/node/lmdb/account_store.hpp +++ /dev/null @@ -1,55 +0,0 @@ -#pragma once - -#include - -#include - -namespace nano -{ -namespace lmdb -{ - class store; - class account_store : public nano::account_store - { - private: - nano::lmdb::store & store; - - public: - explicit account_store (nano::lmdb::store & store_a); - void put (nano::write_transaction const & transaction, nano::account const & account, nano::account_info const & info) override; - bool get (nano::transaction const & transaction_a, nano::account const & account_a, nano::account_info & info_a) override; - void del (nano::write_transaction const & transaction_a, nano::account const & account_a) override; - bool exists (nano::transaction const & transaction_a, nano::account const & account_a) override; - size_t count (nano::transaction const & transaction_a) override; - nano::store_iterator begin (nano::transaction const & transaction_a, nano::account const & account_a) const override; - nano::store_iterator begin (nano::transaction const & transaction_a) const override; - nano::store_iterator rbegin (nano::transaction const & transaction_a) const override; - nano::store_iterator end () const override; - void for_each_par (std::function, nano::store_iterator)> const & action_a) const override; - - /** - * Maps account v1 to account information, head, rep, open, balance, timestamp and block count. (Removed) - * nano::account -> nano::block_hash, nano::block_hash, nano::block_hash, nano::amount, uint64_t, uint64_t - */ - MDB_dbi accounts_v0_handle{ 0 }; - - /** - * Maps account v0 to account information, head, rep, open, balance, timestamp and block count. (Removed) - * nano::account -> nano::block_hash, nano::block_hash, nano::block_hash, nano::amount, uint64_t, uint64_t - */ - MDB_dbi accounts_v1_handle{ 0 }; - - /** - * Maps account v0 to account information, head, rep, open, balance, timestamp, block count and epoch - * nano::account -> nano::block_hash, nano::block_hash, nano::block_hash, nano::amount, uint64_t, uint64_t, nano::epoch - */ - MDB_dbi accounts_handle{ 0 }; - - /** - * Representative weights. (Removed) - * nano::account -> nano::uint128_t - */ - MDB_dbi representation_handle{ 0 }; - }; -} -} diff --git a/nano/node/lmdb/block_store.cpp b/nano/node/lmdb/block_store.cpp deleted file mode 100644 index b82718afd7..0000000000 --- a/nano/node/lmdb/block_store.cpp +++ /dev/null @@ -1,297 +0,0 @@ -#include -#include -#include - -namespace nano -{ -/** - * Fill in our predecessors - */ -class block_predecessor_mdb_set : public nano::block_visitor -{ -public: - block_predecessor_mdb_set (nano::write_transaction const & transaction_a, nano::lmdb::block_store & block_store_a); - virtual ~block_predecessor_mdb_set () = default; - void fill_value (nano::block const & block_a); - void send_block (nano::send_block const & block_a) override; - void receive_block (nano::receive_block const & block_a) override; - void open_block (nano::open_block const & block_a) override; - void change_block (nano::change_block const & block_a) override; - void state_block (nano::state_block const & block_a) override; - nano::write_transaction const & transaction; - nano::lmdb::block_store & block_store; -}; -} - -nano::lmdb::block_store::block_store (nano::lmdb::store & store_a) : - store{ store_a } {}; - -void nano::lmdb::block_store::put (nano::write_transaction const & transaction, nano::block_hash const & hash, nano::block const & block) -{ - debug_assert (block.sideband ().successor.is_zero () || exists (transaction, block.sideband ().successor)); - std::vector vector; - { - nano::vectorstream stream (vector); - nano::serialize_block (stream, block); - block.sideband ().serialize (stream, block.type ()); - } - raw_put (transaction, vector, hash); - block_predecessor_mdb_set predecessor (transaction, *this); - block.visit (predecessor); - debug_assert (block.previous ().is_zero () || successor (transaction, block.previous ()) == hash); -} - -void nano::lmdb::block_store::raw_put (nano::write_transaction const & transaction_a, std::vector const & data, nano::block_hash const & hash_a) -{ - nano::mdb_val value{ data.size (), (void *)data.data () }; - auto status = store.put (transaction_a, tables::blocks, hash_a, value); - store.release_assert_success (status); -} - -nano::block_hash nano::lmdb::block_store::successor (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const -{ - nano::mdb_val value; - block_raw_get (transaction_a, hash_a, value); - nano::block_hash result; - if (value.size () != 0) - { - debug_assert (value.size () >= result.bytes.size ()); - auto type = block_type_from_raw (value.data ()); - nano::bufferstream stream (reinterpret_cast (value.data ()) + block_successor_offset (transaction_a, value.size (), type), result.bytes.size ()); - auto error (nano::try_read (stream, result.bytes)); - (void)error; - debug_assert (!error); - } - else - { - result.clear (); - } - return result; -} - -void nano::lmdb::block_store::successor_clear (nano::write_transaction const & transaction, nano::block_hash const & hash) -{ - nano::mdb_val value; - block_raw_get (transaction, hash, value); - debug_assert (value.size () != 0); - auto type = block_type_from_raw (value.data ()); - std::vector data (static_cast (value.data ()), static_cast (value.data ()) + value.size ()); - std::fill_n (data.begin () + block_successor_offset (transaction, value.size (), type), sizeof (nano::block_hash), uint8_t{ 0 }); - raw_put (transaction, data, hash); -} - -std::shared_ptr nano::lmdb::block_store::get (nano::transaction const & transaction, nano::block_hash const & hash) const -{ - nano::mdb_val value; - block_raw_get (transaction, hash, value); - std::shared_ptr result; - if (value.size () != 0) - { - nano::bufferstream stream (reinterpret_cast (value.data ()), value.size ()); - nano::block_type type; - auto error (try_read (stream, type)); - release_assert (!error); - result = nano::deserialize_block (stream, type); - release_assert (result != nullptr); - nano::block_sideband sideband; - error = (sideband.deserialize (stream, type)); - release_assert (!error); - result->sideband_set (sideband); - } - return result; -} - -std::shared_ptr nano::lmdb::block_store::get_no_sideband (nano::transaction const & transaction, nano::block_hash const & hash) const -{ - nano::mdb_val value; - block_raw_get (transaction, hash, value); - std::shared_ptr result; - if (value.size () != 0) - { - nano::bufferstream stream (reinterpret_cast (value.data ()), value.size ()); - result = nano::deserialize_block (stream); - debug_assert (result != nullptr); - } - return result; -} - -std::shared_ptr nano::lmdb::block_store::random (nano::transaction const & transaction) -{ - nano::block_hash hash; - nano::random_pool::generate_block (hash.bytes.data (), hash.bytes.size ()); - auto existing = begin (transaction, hash); - if (existing == end ()) - { - existing = begin (transaction); - } - debug_assert (existing != end ()); - return existing->second.block; -} - -void nano::lmdb::block_store::del (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a) -{ - auto status = store.del (transaction_a, tables::blocks, hash_a); - store.release_assert_success (status); -} - -bool nano::lmdb::block_store::exists (nano::transaction const & transaction, nano::block_hash const & hash) -{ - nano::mdb_val junk; - block_raw_get (transaction, hash, junk); - return junk.size () != 0; -} - -uint64_t nano::lmdb::block_store::count (nano::transaction const & transaction_a) -{ - return store.count (transaction_a, tables::blocks); -} - -nano::account nano::lmdb::block_store::account (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const -{ - auto block (get (transaction_a, hash_a)); - debug_assert (block != nullptr); - return account_calculated (*block); -} - -nano::account nano::lmdb::block_store::account_calculated (nano::block const & block_a) const -{ - debug_assert (block_a.has_sideband ()); - nano::account result (block_a.account ()); - if (result.is_zero ()) - { - result = block_a.sideband ().account; - } - debug_assert (!result.is_zero ()); - return result; -} - -nano::store_iterator nano::lmdb::block_store::begin (nano::transaction const & transaction) const -{ - return store.make_iterator (transaction, tables::blocks); -} - -nano::store_iterator nano::lmdb::block_store::begin (nano::transaction const & transaction, nano::block_hash const & hash) const -{ - return store.make_iterator (transaction, tables::blocks, hash); -} - -nano::store_iterator nano::lmdb::block_store::end () const -{ - return nano::store_iterator (nullptr); -} - -nano::uint128_t nano::lmdb::block_store::balance (nano::transaction const & transaction_a, nano::block_hash const & hash_a) -{ - auto block (get (transaction_a, hash_a)); - release_assert (block); - nano::uint128_t result (balance_calculated (block)); - return result; -} - -nano::uint128_t nano::lmdb::block_store::balance_calculated (std::shared_ptr const & block_a) const -{ - nano::uint128_t result; - switch (block_a->type ()) - { - case nano::block_type::open: - case nano::block_type::receive: - case nano::block_type::change: - result = block_a->sideband ().balance.number (); - break; - case nano::block_type::send: - result = boost::polymorphic_downcast (block_a.get ())->hashables.balance.number (); - break; - case nano::block_type::state: - result = boost::polymorphic_downcast (block_a.get ())->hashables.balance.number (); - break; - case nano::block_type::invalid: - case nano::block_type::not_a_block: - release_assert (false); - break; - } - return result; -} - -nano::epoch nano::lmdb::block_store::version (nano::transaction const & transaction_a, nano::block_hash const & hash_a) -{ - auto block = get (transaction_a, hash_a); - if (block && block->type () == nano::block_type::state) - { - return block->sideband ().details.epoch; - } - - return nano::epoch::epoch_0; -} - -void nano::lmdb::block_store::for_each_par (std::function, nano::store_iterator)> const & action_a) const -{ - parallel_traversal ( - [&action_a, this] (nano::uint256_t const & start, nano::uint256_t const & end, bool const is_last) { - auto transaction (this->store.tx_begin_read ()); - action_a (transaction, this->begin (transaction, start), !is_last ? this->begin (transaction, end) : this->end ()); - }); -} - -// Converts a block hash to a block height -uint64_t nano::lmdb::block_store::account_height (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const -{ - auto block = get (transaction_a, hash_a); - return block->sideband ().height; -} - -void nano::lmdb::block_store::block_raw_get (nano::transaction const & transaction, nano::block_hash const & hash, nano::mdb_val & value) const -{ - auto status = store.get (transaction, tables::blocks, hash, value); - release_assert (store.success (status) || store.not_found (status)); -} - -size_t nano::lmdb::block_store::block_successor_offset (nano::transaction const & transaction_a, size_t entry_size_a, nano::block_type type_a) const -{ - return entry_size_a - nano::block_sideband::size (type_a); -} - -nano::block_type nano::lmdb::block_store::block_type_from_raw (void * data_a) -{ - // The block type is the first byte - return static_cast ((reinterpret_cast (data_a))[0]); -} - -nano::block_predecessor_mdb_set::block_predecessor_mdb_set (nano::write_transaction const & transaction_a, nano::lmdb::block_store & block_store_a) : - transaction{ transaction_a }, - block_store{ block_store_a } -{ -} -void nano::block_predecessor_mdb_set::fill_value (nano::block const & block_a) -{ - auto hash = block_a.hash (); - nano::mdb_val value; - block_store.block_raw_get (transaction, block_a.previous (), value); - debug_assert (value.size () != 0); - auto type = block_store.block_type_from_raw (value.data ()); - std::vector data (static_cast (value.data ()), static_cast (value.data ()) + value.size ()); - std::copy (hash.bytes.begin (), hash.bytes.end (), data.begin () + block_store.block_successor_offset (transaction, value.size (), type)); - block_store.raw_put (transaction, data, block_a.previous ()); -} -void nano::block_predecessor_mdb_set::send_block (nano::send_block const & block_a) -{ - fill_value (block_a); -} -void nano::block_predecessor_mdb_set::receive_block (nano::receive_block const & block_a) -{ - fill_value (block_a); -} -void nano::block_predecessor_mdb_set::open_block (nano::open_block const & block_a) -{ - // Open blocks don't have a predecessor -} -void nano::block_predecessor_mdb_set::change_block (nano::change_block const & block_a) -{ - fill_value (block_a); -} -void nano::block_predecessor_mdb_set::state_block (nano::state_block const & block_a) -{ - if (!block_a.previous ().is_zero ()) - { - fill_value (block_a); - } -} diff --git a/nano/node/lmdb/block_store.hpp b/nano/node/lmdb/block_store.hpp deleted file mode 100644 index f3169a18cc..0000000000 --- a/nano/node/lmdb/block_store.hpp +++ /dev/null @@ -1,103 +0,0 @@ -#pragma once - -#include - -#include - -namespace nano -{ -using mdb_val = db_val; -class block_predecessor_mdb_set; -namespace lmdb -{ - class store; - class block_store : public nano::block_store - { - friend class nano::block_predecessor_mdb_set; - nano::lmdb::store & store; - - public: - explicit block_store (nano::lmdb::store & store_a); - void put (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a, nano::block const & block_a) override; - void raw_put (nano::write_transaction const & transaction_a, std::vector const & data, nano::block_hash const & hash_a) override; - nano::block_hash successor (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const override; - void successor_clear (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a) override; - std::shared_ptr get (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const override; - std::shared_ptr get_no_sideband (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const override; - std::shared_ptr random (nano::transaction const & transaction_a) override; - void del (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a) override; - bool exists (nano::transaction const & transaction_a, nano::block_hash const & hash_a) override; - uint64_t count (nano::transaction const & transaction_a) override; - nano::account account (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const override; - nano::account account_calculated (nano::block const & block_a) const override; - nano::store_iterator begin (nano::transaction const & transaction_a) const override; - nano::store_iterator begin (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const override; - nano::store_iterator end () const override; - nano::uint128_t balance (nano::transaction const & transaction_a, nano::block_hash const & hash_a) override; - nano::uint128_t balance_calculated (std::shared_ptr const & block_a) const override; - nano::epoch version (nano::transaction const & transaction_a, nano::block_hash const & hash_a) override; - void for_each_par (std::function, nano::store_iterator)> const & action_a) const override; - // Converts a block hash to a block height - uint64_t account_height (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const override; - - /** - * Maps block hash to send block. (Removed) - * nano::block_hash -> nano::send_block - */ - MDB_dbi send_blocks_handle{ 0 }; - - /** - * Maps block hash to receive block. (Removed) - * nano::block_hash -> nano::receive_block - */ - MDB_dbi receive_blocks_handle{ 0 }; - - /** - * Maps block hash to open block. (Removed) - * nano::block_hash -> nano::open_block - */ - MDB_dbi open_blocks_handle{ 0 }; - - /** - * Maps block hash to change block. (Removed) - * nano::block_hash -> nano::change_block - */ - MDB_dbi change_blocks_handle{ 0 }; - - /** - * Maps block hash to v0 state block. (Removed) - * nano::block_hash -> nano::state_block - */ - MDB_dbi state_blocks_v0_handle{ 0 }; - - /** - * Maps block hash to v1 state block. (Removed) - * nano::block_hash -> nano::state_block - */ - MDB_dbi state_blocks_v1_handle{ 0 }; - - /** - * Maps block hash to state block. (Removed) - * nano::block_hash -> nano::state_block - */ - MDB_dbi state_blocks_handle{ 0 }; - - /** - * Meta information about block store, such as versions. - * nano::uint256_union (arbitrary key) -> blob - */ - MDB_dbi meta_handle{ 0 }; - - /** - * Contains block_sideband and block for all block types (legacy send/change/open/receive & state blocks) - * nano::block_hash -> nano::block_sideband, nano::block - */ - MDB_dbi blocks_handle{ 0 }; - - protected: - void block_raw_get (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::mdb_val & value) const; - size_t block_successor_offset (nano::transaction const & transaction_a, size_t entry_size_a, nano::block_type type_a) const; - static nano::block_type block_type_from_raw (void * data_a); - }; -} -} diff --git a/nano/node/lmdb/confirmation_height_store.cpp b/nano/node/lmdb/confirmation_height_store.cpp deleted file mode 100644 index 139b263a43..0000000000 --- a/nano/node/lmdb/confirmation_height_store.cpp +++ /dev/null @@ -1,84 +0,0 @@ -#include -#include -#include - -nano::lmdb::confirmation_height_store::confirmation_height_store (nano::lmdb::store & store) : - store{ store } -{ -} - -void nano::lmdb::confirmation_height_store::put (nano::write_transaction const & transaction, nano::account const & account, nano::confirmation_height_info const & confirmation_height_info) -{ - auto status = store.put (transaction, tables::confirmation_height, account, confirmation_height_info); - store.release_assert_success (status); -} - -bool nano::lmdb::confirmation_height_store::get (nano::transaction const & transaction, nano::account const & account, nano::confirmation_height_info & confirmation_height_info) -{ - nano::mdb_val value; - auto status = store.get (transaction, tables::confirmation_height, account, value); - release_assert (store.success (status) || store.not_found (status)); - bool result (true); - if (store.success (status)) - { - nano::bufferstream stream (reinterpret_cast (value.data ()), value.size ()); - result = confirmation_height_info.deserialize (stream); - } - if (result) - { - confirmation_height_info.height = 0; - confirmation_height_info.frontier = 0; - } - - return result; -} - -bool nano::lmdb::confirmation_height_store::exists (nano::transaction const & transaction, nano::account const & account) const -{ - return store.exists (transaction, tables::confirmation_height, account); -} - -void nano::lmdb::confirmation_height_store::del (nano::write_transaction const & transaction, nano::account const & account) -{ - auto status = store.del (transaction, tables::confirmation_height, account); - store.release_assert_success (status); -} - -uint64_t nano::lmdb::confirmation_height_store::count (nano::transaction const & transaction_a) -{ - return store.count (transaction_a, tables::confirmation_height); -} - -void nano::lmdb::confirmation_height_store::clear (nano::write_transaction const & transaction_a, nano::account const & account_a) -{ - del (transaction_a, account_a); -} - -void nano::lmdb::confirmation_height_store::clear (nano::write_transaction const & transaction_a) -{ - store.drop (transaction_a, nano::tables::confirmation_height); -} - -nano::store_iterator nano::lmdb::confirmation_height_store::begin (nano::transaction const & transaction, nano::account const & account) const -{ - return store.make_iterator (transaction, tables::confirmation_height, account); -} - -nano::store_iterator nano::lmdb::confirmation_height_store::begin (nano::transaction const & transaction) const -{ - return store.make_iterator (transaction, tables::confirmation_height); -} - -nano::store_iterator nano::lmdb::confirmation_height_store::end () const -{ - return nano::store_iterator (nullptr); -} - -void nano::lmdb::confirmation_height_store::for_each_par (std::function, nano::store_iterator)> const & action_a) const -{ - parallel_traversal ( - [&action_a, this] (nano::uint256_t const & start, nano::uint256_t const & end, bool const is_last) { - auto transaction (this->store.tx_begin_read ()); - action_a (transaction, this->begin (transaction, start), !is_last ? this->begin (transaction, end) : this->end ()); - }); -} diff --git a/nano/node/lmdb/confirmation_height_store.hpp b/nano/node/lmdb/confirmation_height_store.hpp deleted file mode 100644 index ed8149942e..0000000000 --- a/nano/node/lmdb/confirmation_height_store.hpp +++ /dev/null @@ -1,37 +0,0 @@ -#pragma once - -#include - -#include - -namespace nano -{ -namespace lmdb -{ - class store; - class confirmation_height_store : public nano::confirmation_height_store - { - nano::lmdb::store & store; - - public: - explicit confirmation_height_store (nano::lmdb::store & store_a); - void put (nano::write_transaction const & transaction_a, nano::account const & account_a, nano::confirmation_height_info const & confirmation_height_info_a) override; - bool get (nano::transaction const & transaction_a, nano::account const & account_a, nano::confirmation_height_info & confirmation_height_info_a) override; - bool exists (nano::transaction const & transaction_a, nano::account const & account_a) const override; - void del (nano::write_transaction const & transaction_a, nano::account const & account_a) override; - uint64_t count (nano::transaction const & transaction_a) override; - void clear (nano::write_transaction const & transaction_a, nano::account const & account_a) override; - void clear (nano::write_transaction const & transaction_a) override; - nano::store_iterator begin (nano::transaction const & transaction_a, nano::account const & account_a) const override; - nano::store_iterator begin (nano::transaction const & transaction_a) const override; - nano::store_iterator end () const override; - void for_each_par (std::function, nano::store_iterator)> const & action_a) const override; - - /* - * Confirmation height of an account, and the hash for the block at that height - * nano::account -> uint64_t, nano::block_hash - */ - MDB_dbi confirmation_height_handle{ 0 }; - }; -} -} diff --git a/nano/node/lmdb/final_vote_store.hpp b/nano/node/lmdb/final_vote_store.hpp deleted file mode 100644 index 8e95764136..0000000000 --- a/nano/node/lmdb/final_vote_store.hpp +++ /dev/null @@ -1,37 +0,0 @@ -#pragma once - -#include - -#include - -namespace nano -{ -namespace lmdb -{ - class store; - class final_vote_store : public nano::final_vote_store - { - private: - nano::lmdb::store & store; - - public: - explicit final_vote_store (nano::lmdb::store & store); - bool put (nano::write_transaction const & transaction_a, nano::qualified_root const & root_a, nano::block_hash const & hash_a) override; - std::vector get (nano::transaction const & transaction_a, nano::root const & root_a) override; - void del (nano::write_transaction const & transaction_a, nano::root const & root_a) override; - size_t count (nano::transaction const & transaction_a) const override; - void clear (nano::write_transaction const & transaction_a, nano::root const & root_a) override; - void clear (nano::write_transaction const & transaction_a) override; - nano::store_iterator begin (nano::transaction const & transaction_a, nano::qualified_root const & root_a) const override; - nano::store_iterator begin (nano::transaction const & transaction_a) const override; - nano::store_iterator end () const override; - void for_each_par (std::function, nano::store_iterator)> const & action_a) const override; - - /** - * Maps root to block hash for generated final votes. - * nano::qualified_root -> nano::block_hash - */ - MDB_dbi final_votes_handle{ 0 }; - }; -} -} diff --git a/nano/node/lmdb/frontier_store.cpp b/nano/node/lmdb/frontier_store.cpp deleted file mode 100644 index f10ff9ffa5..0000000000 --- a/nano/node/lmdb/frontier_store.cpp +++ /dev/null @@ -1,57 +0,0 @@ -#include -#include -#include - -nano::lmdb::frontier_store::frontier_store (nano::lmdb::store & store) : - store{ store } -{ -} - -void nano::lmdb::frontier_store::put (nano::write_transaction const & transaction, nano::block_hash const & hash, nano::account const & account) -{ - auto status = store.put (transaction, tables::frontiers, hash, account); - store.release_assert_success (status); -} - -nano::account nano::lmdb::frontier_store::get (nano::transaction const & transaction, nano::block_hash const & hash) const -{ - nano::db_val value; - auto status = store.get (transaction, tables::frontiers, hash, value); - release_assert (store.success (status) || store.not_found (status)); - nano::account result{}; - if (store.success (status)) - { - result = static_cast (value); - } - return result; -} - -void nano::lmdb::frontier_store::del (nano::write_transaction const & transaction, nano::block_hash const & hash) -{ - auto status = store.del (transaction, tables::frontiers, hash); - store.release_assert_success (status); -} - -nano::store_iterator nano::lmdb::frontier_store::begin (nano::transaction const & transaction) const -{ - return store.make_iterator (transaction, tables::frontiers); -} - -nano::store_iterator nano::lmdb::frontier_store::begin (nano::transaction const & transaction, nano::block_hash const & hash) const -{ - return store.make_iterator (transaction, tables::frontiers, nano::db_val (hash)); -} - -nano::store_iterator nano::lmdb::frontier_store::end () const -{ - return nano::store_iterator (nullptr); -} - -void nano::lmdb::frontier_store::for_each_par (std::function, nano::store_iterator)> const & action_a) const -{ - parallel_traversal ( - [&action_a, this] (nano::uint256_t const & start, nano::uint256_t const & end, bool const is_last) { - auto transaction (this->store.tx_begin_read ()); - action_a (transaction, this->begin (transaction, start), !is_last ? this->begin (transaction, end) : this->end ()); - }); -} diff --git a/nano/node/lmdb/frontier_store.hpp b/nano/node/lmdb/frontier_store.hpp deleted file mode 100644 index faf0d2cf44..0000000000 --- a/nano/node/lmdb/frontier_store.hpp +++ /dev/null @@ -1,34 +0,0 @@ -#pragma once - -#include - -#include - -namespace nano -{ -namespace lmdb -{ - class store; - class frontier_store : public nano::frontier_store - { - private: - nano::lmdb::store & store; - - public: - frontier_store (nano::lmdb::store & store); - void put (nano::write_transaction const &, nano::block_hash const &, nano::account const &) override; - nano::account get (nano::transaction const &, nano::block_hash const &) const override; - void del (nano::write_transaction const &, nano::block_hash const &) override; - nano::store_iterator begin (nano::transaction const &) const override; - nano::store_iterator begin (nano::transaction const &, nano::block_hash const &) const override; - nano::store_iterator end () const override; - void for_each_par (std::function, nano::store_iterator)> const & action_a) const override; - - /** - * Maps head block to owning account - * nano::block_hash -> nano::account - */ - MDB_dbi frontiers_handle{ 0 }; - }; -} -} diff --git a/nano/node/lmdb/lmdb.cpp b/nano/node/lmdb/lmdb.cpp deleted file mode 100644 index 4c28763d46..0000000000 --- a/nano/node/lmdb/lmdb.cpp +++ /dev/null @@ -1,1208 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include - -namespace nano -{ -template <> -void * mdb_val::data () const -{ - return value.mv_data; -} - -template <> -std::size_t mdb_val::size () const -{ - return value.mv_size; -} - -template <> -mdb_val::db_val (std::size_t size_a, void * data_a) : - value ({ size_a, data_a }) -{ -} - -template <> -void mdb_val::convert_buffer_to_value () -{ - value = { buffer->size (), const_cast (buffer->data ()) }; -} -} - -nano::lmdb::store::store (nano::logger_mt & logger_a, boost::filesystem::path const & path_a, nano::ledger_constants & constants, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a, nano::lmdb_config const & lmdb_config_a, bool backup_before_upgrade_a) : - // clang-format off - nano::store{ - block_store, - frontier_store, - account_store, - pending_store, - online_weight_store, - pruned_store, - peer_store, - confirmation_height_store, - final_vote_store, - version_store - }, - // clang-format on - block_store{ *this }, - frontier_store{ *this }, - account_store{ *this }, - pending_store{ *this }, - online_weight_store{ *this }, - pruned_store{ *this }, - peer_store{ *this }, - confirmation_height_store{ *this }, - final_vote_store{ *this }, - version_store{ *this }, - logger (logger_a), - env (error, path_a, nano::mdb_env::options::make ().set_config (lmdb_config_a).set_use_no_mem_init (true)), - mdb_txn_tracker (logger_a, txn_tracking_config_a, block_processor_batch_max_time_a), - txn_tracking_enabled (txn_tracking_config_a.enable) -{ - if (!error) - { - debug_assert (path_a.filename () == "data.ldb"); - - auto is_fully_upgraded (false); - auto is_fresh_db (false); - { - auto transaction (tx_begin_read ()); - auto err = mdb_dbi_open (env.tx (transaction), "meta", 0, &block_store.meta_handle); - is_fresh_db = err != MDB_SUCCESS; - if (err == MDB_SUCCESS) - { - is_fully_upgraded = (version.get (transaction) == version_current); - mdb_dbi_close (env, block_store.meta_handle); - } - } - - // Only open a write lock when upgrades are needed. This is because CLI commands - // open inactive nodes which can otherwise be locked here if there is a long write - // (can be a few minutes with the --fast_bootstrap flag for instance) - if (!is_fully_upgraded) - { - if (!is_fresh_db) - { - logger.always_log ("Upgrade in progress..."); - if (backup_before_upgrade_a) - { - create_backup_file (env, path_a, logger_a); - } - } - auto needs_vacuuming = false; - { - auto transaction (tx_begin_write ()); - open_databases (error, transaction, MDB_CREATE); - if (!error) - { - error |= do_upgrades (transaction, constants, needs_vacuuming); - } - } - - if (needs_vacuuming) - { - logger.always_log ("Preparing vacuum..."); - auto vacuum_success = vacuum_after_upgrade (path_a, lmdb_config_a); - logger.always_log (vacuum_success ? "Vacuum succeeded." : "Failed to vacuum. (Optional) Ensure enough disk space is available for a copy of the database and try to vacuum after shutting down the node"); - } - } - else - { - auto transaction (tx_begin_read ()); - open_databases (error, transaction, 0); - } - } -} - -bool nano::lmdb::store::vacuum_after_upgrade (boost::filesystem::path const & path_a, nano::lmdb_config const & lmdb_config_a) -{ - // Vacuum the database. This is not a required step and may actually fail if there isn't enough storage space. - auto vacuum_path = path_a.parent_path () / "vacuumed.ldb"; - - auto vacuum_success = copy_db (vacuum_path); - if (vacuum_success) - { - // Need to close the database to release the file handle - mdb_env_sync (env.environment, true); - mdb_env_close (env.environment); - env.environment = nullptr; - - // Replace the ledger file with the vacuumed one - boost::filesystem::rename (vacuum_path, path_a); - - // Set up the environment again - auto options = nano::mdb_env::options::make () - .set_config (lmdb_config_a) - .set_use_no_mem_init (true); - env.init (error, path_a, options); - if (!error) - { - auto transaction (tx_begin_read ()); - open_databases (error, transaction, 0); - } - } - else - { - // The vacuum file can be in an inconsistent state if there wasn't enough space to create it - boost::filesystem::remove (vacuum_path); - } - return vacuum_success; -} - -void nano::lmdb::store::serialize_mdb_tracker (boost::property_tree::ptree & json, std::chrono::milliseconds min_read_time, std::chrono::milliseconds min_write_time) -{ - mdb_txn_tracker.serialize_json (json, min_read_time, min_write_time); -} - -void nano::lmdb::store::serialize_memory_stats (boost::property_tree::ptree & json) -{ - MDB_stat stats; - auto status (mdb_env_stat (env.environment, &stats)); - release_assert (status == 0); - json.put ("branch_pages", stats.ms_branch_pages); - json.put ("depth", stats.ms_depth); - json.put ("entries", stats.ms_entries); - json.put ("leaf_pages", stats.ms_leaf_pages); - json.put ("overflow_pages", stats.ms_overflow_pages); - json.put ("page_size", stats.ms_psize); -} - -nano::write_transaction nano::lmdb::store::tx_begin_write (std::vector const &, std::vector const &) -{ - return env.tx_begin_write (create_txn_callbacks ()); -} - -nano::read_transaction nano::lmdb::store::tx_begin_read () const -{ - return env.tx_begin_read (create_txn_callbacks ()); -} - -std::string nano::lmdb::store::vendor_get () const -{ - return boost::str (boost::format ("LMDB %1%.%2%.%3%") % MDB_VERSION_MAJOR % MDB_VERSION_MINOR % MDB_VERSION_PATCH); -} - -nano::mdb_txn_callbacks nano::lmdb::store::create_txn_callbacks () const -{ - nano::mdb_txn_callbacks mdb_txn_callbacks; - if (txn_tracking_enabled) - { - mdb_txn_callbacks.txn_start = ([&mdb_txn_tracker = mdb_txn_tracker] (nano::transaction_impl const * transaction_impl) { - mdb_txn_tracker.add (transaction_impl); - }); - mdb_txn_callbacks.txn_end = ([&mdb_txn_tracker = mdb_txn_tracker] (nano::transaction_impl const * transaction_impl) { - mdb_txn_tracker.erase (transaction_impl); - }); - } - return mdb_txn_callbacks; -} - -void nano::lmdb::store::open_databases (bool & error_a, nano::transaction const & transaction_a, unsigned flags) -{ - error_a |= mdb_dbi_open (env.tx (transaction_a), "frontiers", flags, &frontier_store.frontiers_handle) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "online_weight", flags, &online_weight_store.online_weight_handle) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "meta", flags, &block_store.meta_handle) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "peers", flags, &peer_store.peers_handle) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "pruned", flags, &pruned_store.pruned_handle) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "confirmation_height", flags, &confirmation_height_store.confirmation_height_handle) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "accounts", flags, &account_store.accounts_v0_handle) != 0; - account_store.accounts_handle = account_store.accounts_v0_handle; - error_a |= mdb_dbi_open (env.tx (transaction_a), "pending", flags, &pending_store.pending_v0_handle) != 0; - pending_store.pending_handle = pending_store.pending_v0_handle; - error_a |= mdb_dbi_open (env.tx (transaction_a), "final_votes", flags, &final_vote_store.final_votes_handle) != 0; - - auto version_l = version.get (transaction_a); - if (version_l < 19) - { - // These legacy (and state) block databases are no longer used, but need opening so they can be deleted during an upgrade - error_a |= mdb_dbi_open (env.tx (transaction_a), "send", flags, &block_store.send_blocks_handle) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "receive", flags, &block_store.receive_blocks_handle) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "open", flags, &block_store.open_blocks_handle) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "change", flags, &block_store.change_blocks_handle) != 0; - if (version_l >= 15) - { - error_a |= mdb_dbi_open (env.tx (transaction_a), "state_blocks", flags, &block_store.state_blocks_handle) != 0; - block_store.state_blocks_v0_handle = block_store.state_blocks_handle; - } - } - else - { - error_a |= mdb_dbi_open (env.tx (transaction_a), "blocks", MDB_CREATE, &block_store.blocks_handle) != 0; - } - - if (version_l < 16) - { - // The representation database is no longer used, but needs opening so that it can be deleted during an upgrade - error_a |= mdb_dbi_open (env.tx (transaction_a), "representation", flags, &account_store.representation_handle) != 0; - } - - if (version_l < 15) - { - // These databases are no longer used, but need opening so they can be deleted during an upgrade - error_a |= mdb_dbi_open (env.tx (transaction_a), "state", flags, &block_store.state_blocks_v0_handle) != 0; - block_store.state_blocks_handle = block_store.state_blocks_v0_handle; - error_a |= mdb_dbi_open (env.tx (transaction_a), "accounts_v1", flags, &account_store.accounts_v1_handle) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "pending_v1", flags, &pending_store.pending_v1_handle) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "state_v1", flags, &block_store.state_blocks_v1_handle) != 0; - } -} - -bool nano::lmdb::store::do_upgrades (nano::write_transaction & transaction_a, nano::ledger_constants & constants, bool & needs_vacuuming) -{ - auto error (false); - auto version_l = version.get (transaction_a); - switch (version_l) - { - case 1: - case 2: - case 3: - case 4: - case 5: - case 6: - case 7: - case 8: - case 9: - case 10: - case 11: - case 12: - case 13: - logger.always_log (boost::str (boost::format ("The version of the ledger (%1%) is lower than the minimum (%2%) which is supported for upgrades. Either upgrade to a v19, v20 or v21 node first or delete the ledger.") % version_l % version_minimum)); - error = true; - break; - case 14: - upgrade_v14_to_v15 (transaction_a); - [[fallthrough]]; - // Upgrades to version 16, 17 & 18 are all part of the v21 node release - case 15: - upgrade_v15_to_v16 (transaction_a); - [[fallthrough]]; - case 16: - upgrade_v16_to_v17 (transaction_a); - [[fallthrough]]; - case 17: - upgrade_v17_to_v18 (transaction_a, constants); - [[fallthrough]]; - // Upgrades to version 19 & 20 are both part of the v22 node release - case 18: - upgrade_v18_to_v19 (transaction_a); - needs_vacuuming = true; - [[fallthrough]]; - case 19: - upgrade_v19_to_v20 (transaction_a); - [[fallthrough]]; - case 20: - upgrade_v20_to_v21 (transaction_a); - [[fallthrough]]; - case 21: - upgrade_v21_to_v22 (transaction_a); - [[fallthrough]]; - case 22: - break; - default: - logger.always_log (boost::str (boost::format ("The version of the ledger (%1%) is too high for this node") % version_l)); - error = true; - break; - } - return error; -} - -void nano::lmdb::store::upgrade_v14_to_v15 (nano::write_transaction & transaction_a) -{ - logger.always_log ("Preparing v14 to v15 database upgrade..."); - - std::vector> account_infos; - upgrade_counters account_counters (count (transaction_a, account_store.accounts_v0_handle), count (transaction_a, account_store.accounts_v1_handle)); - account_infos.reserve (account_counters.before_v0 + account_counters.before_v1); - - nano::mdb_merge_iterator i_account (transaction_a, account_store.accounts_v0_handle, account_store.accounts_v1_handle); - nano::mdb_merge_iterator n_account{}; - for (; i_account != n_account; ++i_account) - { - nano::account account (i_account->first); - nano::account_info_v14 account_info_v14 (i_account->second); - - // Upgrade rep block to representative account - auto rep_block = block_get_v14 (transaction_a, account_info_v14.rep_block); - release_assert (rep_block != nullptr); - account_infos.emplace_back (account, nano::account_info{ account_info_v14.head, rep_block->representative (), account_info_v14.open_block, account_info_v14.balance, account_info_v14.modified, account_info_v14.block_count, i_account.from_first_database ? nano::epoch::epoch_0 : nano::epoch::epoch_1 }); - // Move confirmation height from account_info database to its own table - mdb_put (env.tx (transaction_a), confirmation_height_store.confirmation_height_handle, nano::mdb_val (account), nano::mdb_val (account_info_v14.confirmation_height), MDB_APPEND); - i_account.from_first_database ? ++account_counters.after_v0 : ++account_counters.after_v1; - } - - logger.always_log ("Finished extracting confirmation height to its own database"); - - debug_assert (account_counters.are_equal ()); - // No longer need accounts_v1, keep v0 but clear it - mdb_drop (env.tx (transaction_a), account_store.accounts_v1_handle, 1); - mdb_drop (env.tx (transaction_a), account_store.accounts_v0_handle, 0); - - for (auto const & account_account_info_pair : account_infos) - { - auto const & account_info (account_account_info_pair.second); - mdb_put (env.tx (transaction_a), account_store.accounts_handle, nano::mdb_val (account_account_info_pair.first), nano::mdb_val (account_info), MDB_APPEND); - } - - logger.always_log ("Epoch merge upgrade: Finished accounts, now doing state blocks"); - - account_infos.clear (); - - // Have to create a new database as we are iterating over the existing ones and want to use MDB_APPEND for quick insertion - MDB_dbi state_blocks_new; - mdb_dbi_open (env.tx (transaction_a), "state_blocks", MDB_CREATE, &state_blocks_new); - - upgrade_counters state_counters (count (transaction_a, block_store.state_blocks_v0_handle), count (transaction_a, block_store.state_blocks_v1_handle)); - - nano::mdb_merge_iterator i_state (transaction_a, block_store.state_blocks_v0_handle, block_store.state_blocks_v1_handle); - nano::mdb_merge_iterator n_state{}; - auto num = 0u; - for (; i_state != n_state; ++i_state, ++num) - { - nano::block_hash hash (i_state->first); - nano::state_block_w_sideband_v14 state_block_w_sideband_v14 (i_state->second); - auto & sideband_v14 = state_block_w_sideband_v14.sideband; - - nano::block_sideband_v18 sideband (sideband_v14.account, sideband_v14.successor, sideband_v14.balance, sideband_v14.height, sideband_v14.timestamp, i_state.from_first_database ? nano::epoch::epoch_0 : nano::epoch::epoch_1, false, false, false); - - // Write these out - std::vector data; - { - nano::vectorstream stream (data); - state_block_w_sideband_v14.state_block->serialize (stream); - sideband.serialize (stream, sideband_v14.type); - } - - nano::mdb_val value{ data.size (), (void *)data.data () }; - auto s = mdb_put (env.tx (transaction_a), state_blocks_new, nano::mdb_val (hash), value, MDB_APPEND); - release_assert_success (s); - - // Every so often output to the log to indicate progress - constexpr auto output_cutoff = 1000000; - if (num % output_cutoff == 0 && num != 0) - { - logger.always_log (boost::str (boost::format ("Database epoch merge upgrade %1% million state blocks upgraded") % (num / output_cutoff))); - } - i_state.from_first_database ? ++state_counters.after_v0 : ++state_counters.after_v1; - } - - debug_assert (state_counters.are_equal ()); - logger.always_log ("Epoch merge upgrade: Finished state blocks, now doing pending blocks"); - - block_store.state_blocks_handle = state_blocks_new; - - // No longer need states v0/v1 databases - mdb_drop (env.tx (transaction_a), block_store.state_blocks_v1_handle, 1); - mdb_drop (env.tx (transaction_a), block_store.state_blocks_v0_handle, 1); - - block_store.state_blocks_v0_handle = block_store.state_blocks_handle; - - upgrade_counters pending_counters (count (transaction_a, pending_store.pending_v0_handle), count (transaction_a, pending_store.pending_v1_handle)); - std::vector> pending_infos; - pending_infos.reserve (pending_counters.before_v0 + pending_counters.before_v1); - - nano::mdb_merge_iterator i_pending (transaction_a, pending_store.pending_v0_handle, pending_store.pending_v1_handle); - nano::mdb_merge_iterator n_pending{}; - for (; i_pending != n_pending; ++i_pending) - { - nano::pending_info_v14 info (i_pending->second); - pending_infos.emplace_back (nano::pending_key (i_pending->first), nano::pending_info{ info.source, info.amount, i_pending.from_first_database ? nano::epoch::epoch_0 : nano::epoch::epoch_1 }); - i_pending.from_first_database ? ++pending_counters.after_v0 : ++pending_counters.after_v1; - } - - debug_assert (pending_counters.are_equal ()); - - // No longer need the pending v1 table - mdb_drop (env.tx (transaction_a), pending_store.pending_v1_handle, 1); - mdb_drop (env.tx (transaction_a), pending_store.pending_v0_handle, 0); - - for (auto const & pending_key_pending_info_pair : pending_infos) - { - mdb_put (env.tx (transaction_a), pending_store.pending_handle, nano::mdb_val (pending_key_pending_info_pair.first), nano::mdb_val (pending_key_pending_info_pair.second), MDB_APPEND); - } - - version.put (transaction_a, 15); - logger.always_log ("Finished epoch merge upgrade"); -} - -void nano::lmdb::store::upgrade_v15_to_v16 (nano::write_transaction const & transaction_a) -{ - // Representation table is no longer used - debug_assert (account_store.representation_handle != 0); - if (account_store.representation_handle != 0) - { - auto status (mdb_drop (env.tx (transaction_a), account_store.representation_handle, 1)); - release_assert (status == MDB_SUCCESS); - account_store.representation_handle = 0; - } - version.put (transaction_a, 16); -} - -void nano::lmdb::store::upgrade_v16_to_v17 (nano::write_transaction const & transaction_a) -{ - logger.always_log ("Preparing v16 to v17 database upgrade..."); - - auto account_info_i = account.begin (transaction_a); - auto account_info_n = account.end (); - - // Set the confirmed frontier for each account in the confirmation height table - std::vector> confirmation_height_infos; - auto num = 0u; - for (nano::mdb_iterator i (transaction_a, confirmation_height_store.confirmation_height_handle), n (nano::mdb_iterator{}); i != n; ++i, ++account_info_i, ++num) - { - nano::account account (i->first); - uint64_t confirmation_height (i->second); - - // Check account hashes matches both the accounts table and confirmation height table - debug_assert (account == account_info_i->first); - - auto const & account_info = account_info_i->second; - - if (confirmation_height == 0) - { - confirmation_height_infos.emplace_back (account, confirmation_height_info{ 0, nano::block_hash (0) }); - } - else - { - if (account_info_i->second.block_count / 2 >= confirmation_height) - { - // The confirmation height of the account is closer to the bottom of the chain, so start there and work up - auto block = block_get_v18 (transaction_a, account_info.open_block); - debug_assert (block); - auto height = 1; - - while (height != confirmation_height) - { - block = block_get_v18 (transaction_a, block->sideband ().successor); - debug_assert (block); - ++height; - } - - debug_assert (block->sideband ().height == confirmation_height); - confirmation_height_infos.emplace_back (account, confirmation_height_info{ confirmation_height, block->hash () }); - } - else - { - // The confirmation height of the account is closer to the top of the chain so start there and work down - auto block = block_get_v18 (transaction_a, account_info.head); - auto height = block->sideband ().height; - while (height != confirmation_height) - { - block = block_get_v18 (transaction_a, block->previous ()); - debug_assert (block); - --height; - } - confirmation_height_infos.emplace_back (account, confirmation_height_info{ confirmation_height, block->hash () }); - } - } - - // Every so often output to the log to indicate progress (every 200k accounts) - constexpr auto output_cutoff = 200000; - if (num % output_cutoff == 0 && num != 0) - { - logger.always_log (boost::str (boost::format ("Confirmation height frontier set for %1%00k accounts") % ((num / output_cutoff) * 2))); - } - } - - // Clear it then append - auto status (mdb_drop (env.tx (transaction_a), confirmation_height_store.confirmation_height_handle, 0)); - release_assert_success (status); - - for (auto const & confirmation_height_info_pair : confirmation_height_infos) - { - mdb_put (env.tx (transaction_a), confirmation_height_store.confirmation_height_handle, nano::mdb_val (confirmation_height_info_pair.first), nano::mdb_val (confirmation_height_info_pair.second), MDB_APPEND); - } - - version.put (transaction_a, 17); - logger.always_log ("Finished upgrading confirmation height frontiers"); -} - -void nano::lmdb::store::upgrade_v17_to_v18 (nano::write_transaction const & transaction_a, nano::ledger_constants & constants) -{ - logger.always_log ("Preparing v17 to v18 database upgrade..."); - - auto count_pre (count (transaction_a, block_store.state_blocks_handle)); - - auto num = 0u; - for (nano::mdb_iterator> state_i (transaction_a, block_store.state_blocks_handle), state_n{}; state_i != state_n; ++state_i, ++num) - { - nano::block_w_sideband_v18 block_w_sideband (state_i->second); - auto & block (block_w_sideband.block); - auto & sideband (block_w_sideband.sideband); - - bool is_send{ false }; - bool is_receive{ false }; - bool is_epoch{ false }; - - nano::amount prev_balance (0); - if (!block->hashables.previous.is_zero ()) - { - prev_balance = block_balance_v18 (transaction_a, block->hashables.previous); - } - if (block->hashables.balance == prev_balance && constants.epochs.is_epoch_link (block->hashables.link)) - { - is_epoch = true; - } - else if (block->hashables.balance < prev_balance) - { - is_send = true; - } - else if (!block->hashables.link.is_zero ()) - { - is_receive = true; - } - - nano::block_sideband_v18 new_sideband (sideband.account, sideband.successor, sideband.balance, sideband.height, sideband.timestamp, sideband.details.epoch, is_send, is_receive, is_epoch); - // Write these out - std::vector data; - { - nano::vectorstream stream (data); - block->serialize (stream); - new_sideband.serialize (stream, block->type ()); - } - nano::mdb_val value{ data.size (), (void *)data.data () }; - auto s = mdb_cursor_put (state_i.cursor, state_i->first, value, MDB_CURRENT); - release_assert_success (s); - - // Every so often output to the log to indicate progress - constexpr auto output_cutoff = 1000000; - if (num > 0 && num % output_cutoff == 0) - { - logger.always_log (boost::str (boost::format ("Database sideband upgrade %1% million state blocks upgraded (out of %2%)") % (num / output_cutoff) % count_pre)); - } - } - - auto count_post (count (transaction_a, block_store.state_blocks_handle)); - release_assert (count_pre == count_post); - - version.put (transaction_a, 18); - logger.always_log ("Finished upgrading the sideband"); -} - -void nano::lmdb::store::upgrade_v18_to_v19 (nano::write_transaction const & transaction_a) -{ - logger.always_log ("Preparing v18 to v19 database upgrade..."); - auto count_pre (count (transaction_a, block_store.state_blocks_handle) + count (transaction_a, block_store.send_blocks_handle) + count (transaction_a, block_store.receive_blocks_handle) + count (transaction_a, block_store.change_blocks_handle) + count (transaction_a, block_store.open_blocks_handle)); - - // Combine in order of likeliness of counts - std::map legacy_open_receive_change_blocks; - - for (auto i (nano::store_iterator> (std::make_unique>> (transaction_a, block_store.change_blocks_handle))), n (nano::store_iterator> (nullptr)); i != n; ++i) - { - nano::block_sideband_v18 const & old_sideband (i->second.sideband); - nano::block_sideband new_sideband (old_sideband.account, old_sideband.successor, old_sideband.balance, old_sideband.height, old_sideband.timestamp, nano::epoch::epoch_0, false, false, false, nano::epoch::epoch_0); - legacy_open_receive_change_blocks[i->first] = { nano::block_w_sideband{ i->second.block, new_sideband } }; - } - - for (auto i (nano::store_iterator> (std::make_unique>> (transaction_a, block_store.open_blocks_handle))), n (nano::store_iterator> (nullptr)); i != n; ++i) - { - nano::block_sideband_v18 const & old_sideband (i->second.sideband); - nano::block_sideband new_sideband (old_sideband.account, old_sideband.successor, old_sideband.balance, old_sideband.height, old_sideband.timestamp, nano::epoch::epoch_0, false, false, false, nano::epoch::epoch_0); - legacy_open_receive_change_blocks[i->first] = { nano::block_w_sideband{ i->second.block, new_sideband } }; - } - - for (auto i (nano::store_iterator> (std::make_unique>> (transaction_a, block_store.receive_blocks_handle))), n (nano::store_iterator> (nullptr)); i != n; ++i) - { - nano::block_sideband_v18 const & old_sideband (i->second.sideband); - nano::block_sideband new_sideband (old_sideband.account, old_sideband.successor, old_sideband.balance, old_sideband.height, old_sideband.timestamp, nano::epoch::epoch_0, false, false, false, nano::epoch::epoch_0); - legacy_open_receive_change_blocks[i->first] = { nano::block_w_sideband{ i->second.block, new_sideband } }; - } - - release_assert (!mdb_drop (env.tx (transaction_a), block_store.receive_blocks_handle, 1)); - block_store.receive_blocks_handle = 0; - release_assert (!mdb_drop (env.tx (transaction_a), block_store.open_blocks_handle, 1)); - block_store.open_blocks_handle = 0; - release_assert (!mdb_drop (env.tx (transaction_a), block_store.change_blocks_handle, 1)); - block_store.change_blocks_handle = 0; - - logger.always_log ("Write legacy open/receive/change to new format"); - - MDB_dbi temp_legacy_open_receive_change_blocks; - { - mdb_dbi_open (env.tx (transaction_a), "temp_legacy_open_receive_change_blocks", MDB_CREATE, &temp_legacy_open_receive_change_blocks); - - for (auto const & legacy_block : legacy_open_receive_change_blocks) - { - std::vector data; - { - nano::vectorstream stream (data); - nano::serialize_block (stream, *legacy_block.second.block); - legacy_block.second.sideband.serialize (stream, legacy_block.second.block->type ()); - } - - nano::mdb_val value{ data.size (), (void *)data.data () }; - auto s = mdb_put (env.tx (transaction_a), temp_legacy_open_receive_change_blocks, nano::mdb_val (legacy_block.first), value, MDB_APPEND); - release_assert_success (s); - } - } - - logger.always_log ("Write legacy send to new format"); - - // Write send blocks to a new table (this was not done in memory as it would push us above memory requirements) - MDB_dbi temp_legacy_send_blocks; - { - mdb_dbi_open (env.tx (transaction_a), "temp_legacy_send_blocks", MDB_CREATE, &temp_legacy_send_blocks); - - for (auto i (nano::store_iterator> (std::make_unique>> (transaction_a, block_store.send_blocks_handle))), n (nano::store_iterator> (nullptr)); i != n; ++i) - { - auto const & block_w_sideband_v18 (i->second); - - std::vector data; - { - nano::vectorstream stream (data); - nano::serialize_block (stream, *block_w_sideband_v18.block); - block_w_sideband_v18.sideband.serialize (stream, nano::block_type::send); // Equal to new version for legacy blocks - } - - nano::mdb_val value{ data.size (), (void *)data.data () }; - auto s = mdb_put (env.tx (transaction_a), temp_legacy_send_blocks, nano::mdb_val (i->first), value, MDB_APPEND); - release_assert_success (s); - } - } - - release_assert (!mdb_drop (env.tx (transaction_a), block_store.send_blocks_handle, 1)); - block_store.send_blocks_handle = 0; - - logger.always_log ("Merge legacy open/receive/change with legacy send blocks"); - - MDB_dbi temp_legacy_send_open_receive_change_blocks; - { - mdb_dbi_open (env.tx (transaction_a), "temp_legacy_send_open_receive_change_blocks", MDB_CREATE, &temp_legacy_send_open_receive_change_blocks); - - nano::mdb_merge_iterator i (transaction_a, temp_legacy_open_receive_change_blocks, temp_legacy_send_blocks); - nano::mdb_merge_iterator n{}; - for (; i != n; ++i) - { - auto s = mdb_put (env.tx (transaction_a), temp_legacy_send_open_receive_change_blocks, nano::mdb_val (i->first), nano::mdb_val (i->second), MDB_APPEND); - release_assert_success (s); - } - - // Delete tables - mdb_drop (env.tx (transaction_a), temp_legacy_send_blocks, 1); - mdb_drop (env.tx (transaction_a), temp_legacy_open_receive_change_blocks, 1); - } - - logger.always_log ("Write state blocks to new format"); - - // Write state blocks to a new table (this was not done in memory as it would push us above memory requirements) - MDB_dbi temp_state_blocks; - { - auto type_state (nano::block_type::state); - mdb_dbi_open (env.tx (transaction_a), "temp_state_blocks", MDB_CREATE, &temp_state_blocks); - - for (auto i (nano::store_iterator> (std::make_unique>> (transaction_a, block_store.state_blocks_handle))), n (nano::store_iterator> (nullptr)); i != n; ++i) - { - auto const & block_w_sideband_v18 (i->second); - nano::block_sideband_v18 const & old_sideband (block_w_sideband_v18.sideband); - nano::epoch source_epoch (nano::epoch::epoch_0); - // Source block v18 epoch - if (old_sideband.details.is_receive) - { - auto db_val (block_raw_get_by_type_v18 (transaction_a, block_w_sideband_v18.block->link ().as_block_hash (), type_state)); - if (db_val.is_initialized ()) - { - nano::bufferstream stream (reinterpret_cast (db_val.get ().data ()), db_val.get ().size ()); - auto source_block (nano::deserialize_block (stream, type_state)); - release_assert (source_block != nullptr); - nano::block_sideband_v18 source_sideband; - auto error (source_sideband.deserialize (stream, type_state)); - release_assert (!error); - source_epoch = source_sideband.details.epoch; - } - } - nano::block_sideband new_sideband (old_sideband.account, old_sideband.successor, old_sideband.balance, old_sideband.height, old_sideband.timestamp, old_sideband.details.epoch, old_sideband.details.is_send, old_sideband.details.is_receive, old_sideband.details.is_epoch, source_epoch); - - std::vector data; - { - nano::vectorstream stream (data); - nano::serialize_block (stream, *block_w_sideband_v18.block); - new_sideband.serialize (stream, nano::block_type::state); - } - - nano::mdb_val value{ data.size (), (void *)data.data () }; - auto s = mdb_put (env.tx (transaction_a), temp_state_blocks, nano::mdb_val (i->first), value, MDB_APPEND); - release_assert_success (s); - } - } - - release_assert (!mdb_drop (env.tx (transaction_a), block_store.state_blocks_handle, 1)); - block_store.state_blocks_handle = 0; - - logger.always_log ("Merging all legacy blocks with state blocks"); - - // Merge all legacy blocks with state blocks into the final table - nano::mdb_merge_iterator i (transaction_a, temp_legacy_send_open_receive_change_blocks, temp_state_blocks); - nano::mdb_merge_iterator n{}; - mdb_dbi_open (env.tx (transaction_a), "blocks", MDB_CREATE, &block_store.blocks_handle); - for (; i != n; ++i) - { - auto s = mdb_put (env.tx (transaction_a), block_store.blocks_handle, nano::mdb_val (i->first), nano::mdb_val (i->second), MDB_APPEND); - release_assert_success (s); - } - - // Delete tables - mdb_drop (env.tx (transaction_a), temp_legacy_send_open_receive_change_blocks, 1); - mdb_drop (env.tx (transaction_a), temp_state_blocks, 1); - - auto count_post (count (transaction_a, block_store.blocks_handle)); - release_assert (count_pre == count_post); - - MDB_dbi vote{ 0 }; - release_assert (!mdb_dbi_open (env.tx (transaction_a), "vote", MDB_CREATE, &vote)); - release_assert (!mdb_drop (env.tx (transaction_a), vote, 1)); - - version.put (transaction_a, 19); - logger.always_log ("Finished upgrading all blocks to new blocks database"); -} - -void nano::lmdb::store::upgrade_v19_to_v20 (nano::write_transaction const & transaction_a) -{ - logger.always_log ("Preparing v19 to v20 database upgrade..."); - mdb_dbi_open (env.tx (transaction_a), "pruned", MDB_CREATE, &pruned_store.pruned_handle); - version.put (transaction_a, 20); - logger.always_log ("Finished creating new pruned table"); -} - -void nano::lmdb::store::upgrade_v20_to_v21 (nano::write_transaction const & transaction_a) -{ - logger.always_log ("Preparing v20 to v21 database upgrade..."); - mdb_dbi_open (env.tx (transaction_a), "final_votes", MDB_CREATE, &final_vote_store.final_votes_handle); - version.put (transaction_a, 21); - logger.always_log ("Finished creating new final_vote table"); -} - -void nano::lmdb::store::upgrade_v21_to_v22 (nano::write_transaction const & transaction_a) -{ - logger.always_log ("Preparing v21 to v22 database upgrade..."); - MDB_dbi unchecked_handle{ 0 }; - release_assert (!mdb_dbi_open (env.tx (transaction_a), "unchecked", MDB_CREATE, &unchecked_handle)); - release_assert (!mdb_drop (env.tx (transaction_a), unchecked_handle, 1)); // del = 1, to delete it from the environment and close the DB handle. - version.put (transaction_a, 22); - logger.always_log ("Finished removing unchecked table"); -} - -/** Takes a filepath, appends '_backup_' to the end (but before any extension) and saves that file in the same directory */ -void nano::lmdb::store::create_backup_file (nano::mdb_env & env_a, boost::filesystem::path const & filepath_a, nano::logger_mt & logger_a) -{ - auto extension = filepath_a.extension (); - auto filename_without_extension = filepath_a.filename ().replace_extension (""); - auto orig_filepath = filepath_a; - auto & backup_path = orig_filepath.remove_filename (); - auto backup_filename = filename_without_extension; - backup_filename += "_backup_"; - backup_filename += std::to_string (std::chrono::system_clock::now ().time_since_epoch ().count ()); - backup_filename += extension; - auto backup_filepath = backup_path / backup_filename; - auto start_message (boost::str (boost::format ("Performing %1% backup before database upgrade...") % filepath_a.filename ())); - logger_a.always_log (start_message); - std::cout << start_message << std::endl; - auto error (mdb_env_copy (env_a, backup_filepath.string ().c_str ())); - if (error) - { - auto error_message (boost::str (boost::format ("%1% backup failed") % filepath_a.filename ())); - logger_a.always_log (error_message); - std::cerr << error_message << std::endl; - std::exit (1); - } - else - { - auto success_message (boost::str (boost::format ("Backup created: %1%") % backup_filename)); - logger_a.always_log (success_message); - std::cout << success_message << std::endl; - } -} - -bool nano::lmdb::store::exists (nano::transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a) const -{ - nano::mdb_val junk; - auto status = get (transaction_a, table_a, key_a, junk); - release_assert (status == MDB_SUCCESS || status == MDB_NOTFOUND); - return (status == MDB_SUCCESS); -} - -int nano::lmdb::store::get (nano::transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a, nano::mdb_val & value_a) const -{ - return mdb_get (env.tx (transaction_a), table_to_dbi (table_a), key_a, value_a); -} - -int nano::lmdb::store::put (nano::write_transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a, nano::mdb_val const & value_a) const -{ - return (mdb_put (env.tx (transaction_a), table_to_dbi (table_a), key_a, value_a, 0)); -} - -int nano::lmdb::store::del (nano::write_transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a) const -{ - return (mdb_del (env.tx (transaction_a), table_to_dbi (table_a), key_a, nullptr)); -} - -int nano::lmdb::store::drop (nano::write_transaction const & transaction_a, tables table_a) -{ - return clear (transaction_a, table_to_dbi (table_a)); -} - -int nano::lmdb::store::clear (nano::write_transaction const & transaction_a, MDB_dbi handle_a) -{ - return mdb_drop (env.tx (transaction_a), handle_a, 0); -} - -uint64_t nano::lmdb::store::count (nano::transaction const & transaction_a, tables table_a) const -{ - return count (transaction_a, table_to_dbi (table_a)); -} - -uint64_t nano::lmdb::store::count (nano::transaction const & transaction_a, MDB_dbi db_a) const -{ - MDB_stat stats; - auto status (mdb_stat (env.tx (transaction_a), db_a, &stats)); - release_assert_success (status); - return (stats.ms_entries); -} - -MDB_dbi nano::lmdb::store::table_to_dbi (tables table_a) const -{ - switch (table_a) - { - case tables::frontiers: - return frontier_store.frontiers_handle; - case tables::accounts: - return account_store.accounts_handle; - case tables::blocks: - return block_store.blocks_handle; - case tables::pending: - return pending_store.pending_handle; - case tables::online_weight: - return online_weight_store.online_weight_handle; - case tables::meta: - return block_store.meta_handle; - case tables::peers: - return peer_store.peers_handle; - case tables::pruned: - return pruned_store.pruned_handle; - case tables::confirmation_height: - return confirmation_height_store.confirmation_height_handle; - case tables::final_votes: - return final_vote_store.final_votes_handle; - default: - release_assert (false); - return peer_store.peers_handle; - } -} - -bool nano::lmdb::store::not_found (int status) const -{ - return (status_code_not_found () == status); -} - -bool nano::lmdb::store::success (int status) const -{ - return (MDB_SUCCESS == status); -} - -int nano::lmdb::store::status_code_not_found () const -{ - return MDB_NOTFOUND; -} - -std::string nano::lmdb::store::error_string (int status) const -{ - return mdb_strerror (status); -} - -bool nano::lmdb::store::copy_db (boost::filesystem::path const & destination_file) -{ - return !mdb_env_copy2 (env.environment, destination_file.string ().c_str (), MDB_CP_COMPACT); -} - -void nano::lmdb::store::rebuild_db (nano::write_transaction const & transaction_a) -{ - // Tables with uint256_union key - std::vector tables = { account_store.accounts_handle, block_store.blocks_handle, pruned_store.pruned_handle, confirmation_height_store.confirmation_height_handle }; - for (auto const & table : tables) - { - MDB_dbi temp; - mdb_dbi_open (env.tx (transaction_a), "temp_table", MDB_CREATE, &temp); - // Copy all values to temporary table - for (auto i (nano::store_iterator (std::make_unique> (transaction_a, table))), n (nano::store_iterator (nullptr)); i != n; ++i) - { - auto s = mdb_put (env.tx (transaction_a), temp, nano::mdb_val (i->first), i->second, MDB_APPEND); - release_assert_success (s); - } - release_assert (count (transaction_a, table) == count (transaction_a, temp)); - // Clear existing table - mdb_drop (env.tx (transaction_a), table, 0); - // Put values from copy - for (auto i (nano::store_iterator (std::make_unique> (transaction_a, temp))), n (nano::store_iterator (nullptr)); i != n; ++i) - { - auto s = mdb_put (env.tx (transaction_a), table, nano::mdb_val (i->first), i->second, MDB_APPEND); - release_assert_success (s); - } - release_assert (count (transaction_a, table) == count (transaction_a, temp)); - // Remove temporary table - mdb_drop (env.tx (transaction_a), temp, 1); - } - // Pending table - { - MDB_dbi temp; - mdb_dbi_open (env.tx (transaction_a), "temp_table", MDB_CREATE, &temp); - // Copy all values to temporary table - for (auto i (nano::store_iterator (std::make_unique> (transaction_a, pending_store.pending_handle))), n (nano::store_iterator (nullptr)); i != n; ++i) - { - auto s = mdb_put (env.tx (transaction_a), temp, nano::mdb_val (i->first), nano::mdb_val (i->second), MDB_APPEND); - release_assert_success (s); - } - release_assert (count (transaction_a, pending_store.pending_handle) == count (transaction_a, temp)); - mdb_drop (env.tx (transaction_a), pending_store.pending_handle, 0); - // Put values from copy - for (auto i (nano::store_iterator (std::make_unique> (transaction_a, temp))), n (nano::store_iterator (nullptr)); i != n; ++i) - { - auto s = mdb_put (env.tx (transaction_a), pending_store.pending_handle, nano::mdb_val (i->first), nano::mdb_val (i->second), MDB_APPEND); - release_assert_success (s); - } - release_assert (count (transaction_a, pending_store.pending_handle) == count (transaction_a, temp)); - mdb_drop (env.tx (transaction_a), temp, 1); - } -} - -bool nano::lmdb::store::init_error () const -{ - return error; -} - -std::shared_ptr nano::lmdb::store::block_get_v18 (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const -{ - nano::block_type type; - auto value (block_raw_get_v18 (transaction_a, hash_a, type)); - std::shared_ptr result; - if (value.size () != 0) - { - nano::bufferstream stream (reinterpret_cast (value.data ()), value.size ()); - result = nano::deserialize_block (stream, type); - release_assert (result != nullptr); - nano::block_sideband_v18 sideband; - auto error = (sideband.deserialize (stream, type)); - release_assert (!error); - result->sideband_set (nano::block_sideband (sideband.account, sideband.successor, sideband.balance, sideband.height, sideband.timestamp, sideband.details.epoch, sideband.details.is_send, sideband.details.is_receive, sideband.details.is_epoch, nano::epoch::epoch_0)); - } - return result; -} - -nano::mdb_val nano::lmdb::store::block_raw_get_v18 (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type & type_a) const -{ - nano::mdb_val result; - // Table lookups are ordered by match probability - nano::block_type block_types[]{ nano::block_type::state, nano::block_type::send, nano::block_type::receive, nano::block_type::open, nano::block_type::change }; - for (auto current_type : block_types) - { - auto db_val (block_raw_get_by_type_v18 (transaction_a, hash_a, current_type)); - if (db_val.is_initialized ()) - { - type_a = current_type; - result = db_val.get (); - break; - } - } - - return result; -} - -boost::optional nano::lmdb::store::block_raw_get_by_type_v18 (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type & type_a) const -{ - nano::mdb_val value; - nano::mdb_val hash (hash_a); - int status = status_code_not_found (); - switch (type_a) - { - case nano::block_type::send: - { - status = mdb_get (env.tx (transaction_a), block_store.send_blocks_handle, hash, value); - break; - } - case nano::block_type::receive: - { - status = mdb_get (env.tx (transaction_a), block_store.receive_blocks_handle, hash, value); - break; - } - case nano::block_type::open: - { - status = mdb_get (env.tx (transaction_a), block_store.open_blocks_handle, hash, value); - break; - } - case nano::block_type::change: - { - status = mdb_get (env.tx (transaction_a), block_store.change_blocks_handle, hash, value); - break; - } - case nano::block_type::state: - { - status = mdb_get (env.tx (transaction_a), block_store.state_blocks_handle, hash, value); - break; - } - case nano::block_type::invalid: - case nano::block_type::not_a_block: - { - break; - } - } - - release_assert (success (status) || not_found (status)); - boost::optional result; - if (success (status)) - { - result = value; - } - return result; -} - -nano::uint128_t nano::lmdb::store::block_balance_v18 (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const -{ - auto block (block_get_v18 (transaction_a, hash_a)); - release_assert (block); - nano::uint128_t result (this->block.balance_calculated (block)); - return result; -} - -// All the v14 functions below are only needed during upgrades -std::size_t nano::lmdb::store::block_successor_offset_v14 (nano::transaction const & transaction_a, std::size_t entry_size_a, nano::block_type type_a) const -{ - return entry_size_a - nano::block_sideband_v14::size (type_a); -} - -nano::block_hash nano::lmdb::store::block_successor_v14 (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const -{ - nano::block_type type; - auto value (block_raw_get_v14 (transaction_a, hash_a, type)); - nano::block_hash result; - if (value.size () != 0) - { - debug_assert (value.size () >= result.bytes.size ()); - nano::bufferstream stream (reinterpret_cast (value.data ()) + block_successor_offset_v14 (transaction_a, value.size (), type), result.bytes.size ()); - auto error (nano::try_read (stream, result.bytes)); - (void)error; - debug_assert (!error); - } - else - { - result.clear (); - } - return result; -} - -nano::mdb_val nano::lmdb::store::block_raw_get_v14 (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type & type_a, bool * is_state_v1) const -{ - nano::mdb_val result; - // Table lookups are ordered by match probability - nano::block_type block_types[]{ nano::block_type::state, nano::block_type::send, nano::block_type::receive, nano::block_type::open, nano::block_type::change }; - for (auto current_type : block_types) - { - auto db_val (block_raw_get_by_type_v14 (transaction_a, hash_a, current_type, is_state_v1)); - if (db_val.is_initialized ()) - { - type_a = current_type; - result = db_val.get (); - break; - } - } - - return result; -} - -boost::optional nano::lmdb::store::block_raw_get_by_type_v14 (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type & type_a, bool * is_state_v1) const -{ - nano::mdb_val value; - nano::mdb_val hash (hash_a); - int status = status_code_not_found (); - switch (type_a) - { - case nano::block_type::send: - { - status = mdb_get (env.tx (transaction_a), block_store.send_blocks_handle, hash, value); - break; - } - case nano::block_type::receive: - { - status = mdb_get (env.tx (transaction_a), block_store.receive_blocks_handle, hash, value); - break; - } - case nano::block_type::open: - { - status = mdb_get (env.tx (transaction_a), block_store.open_blocks_handle, hash, value); - break; - } - case nano::block_type::change: - { - status = mdb_get (env.tx (transaction_a), block_store.change_blocks_handle, hash, value); - break; - } - case nano::block_type::state: - { - status = mdb_get (env.tx (transaction_a), block_store.state_blocks_v1_handle, hash, value); - if (is_state_v1 != nullptr) - { - *is_state_v1 = success (status); - } - if (not_found (status)) - { - status = mdb_get (env.tx (transaction_a), block_store.state_blocks_v0_handle, hash, value); - } - break; - } - case nano::block_type::invalid: - case nano::block_type::not_a_block: - { - break; - } - } - - release_assert (success (status) || not_found (status)); - boost::optional result; - if (success (status)) - { - result = value; - } - return result; -} - -std::shared_ptr nano::lmdb::store::block_get_v14 (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_sideband_v14 * sideband_a, bool * is_state_v1) const -{ - nano::block_type type; - auto value (block_raw_get_v14 (transaction_a, hash_a, type, is_state_v1)); - std::shared_ptr result; - if (value.size () != 0) - { - nano::bufferstream stream (reinterpret_cast (value.data ()), value.size ()); - result = nano::deserialize_block (stream, type); - debug_assert (result != nullptr); - if (sideband_a) - { - sideband_a->type = type; - bool error = sideband_a->deserialize (stream); - (void)error; - debug_assert (!error); - } - } - return result; -} - -nano::lmdb::store::upgrade_counters::upgrade_counters (uint64_t count_before_v0, uint64_t count_before_v1) : - before_v0 (count_before_v0), - before_v1 (count_before_v1) -{ -} - -bool nano::lmdb::store::upgrade_counters::are_equal () const -{ - return (before_v0 == after_v0) && (before_v1 == after_v1); -} - -unsigned nano::lmdb::store::max_block_write_batch_num () const -{ - return std::numeric_limits::max (); -} diff --git a/nano/node/lmdb/lmdb.hpp b/nano/node/lmdb/lmdb.hpp deleted file mode 100644 index 84c4b432c8..0000000000 --- a/nano/node/lmdb/lmdb.hpp +++ /dev/null @@ -1,206 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -namespace boost -{ -namespace filesystem -{ - class path; -} -} - -namespace nano -{ -using mdb_val = db_val; - -class logging_mt; -class transaction; - -namespace lmdb -{ - /** - * mdb implementation of the block store - */ - class store : public nano::store - { - private: - nano::lmdb::account_store account_store; - nano::lmdb::block_store block_store; - nano::lmdb::confirmation_height_store confirmation_height_store; - nano::lmdb::final_vote_store final_vote_store; - nano::lmdb::frontier_store frontier_store; - nano::lmdb::online_weight_store online_weight_store; - nano::lmdb::peer_store peer_store; - nano::lmdb::pending_store pending_store; - nano::lmdb::pruned_store pruned_store; - nano::lmdb::version_store version_store; - - friend class nano::lmdb::account_store; - friend class nano::lmdb::block_store; - friend class nano::lmdb::confirmation_height_store; - friend class nano::lmdb::final_vote_store; - friend class nano::lmdb::frontier_store; - friend class nano::lmdb::online_weight_store; - friend class nano::lmdb::peer_store; - friend class nano::lmdb::pending_store; - friend class nano::lmdb::pruned_store; - friend class nano::lmdb::version_store; - - public: - store (nano::logger_mt &, boost::filesystem::path const &, nano::ledger_constants & constants, nano::txn_tracking_config const & txn_tracking_config_a = nano::txn_tracking_config{}, std::chrono::milliseconds block_processor_batch_max_time_a = std::chrono::milliseconds (5000), nano::lmdb_config const & lmdb_config_a = nano::lmdb_config{}, bool backup_before_upgrade = false); - nano::write_transaction tx_begin_write (std::vector const & tables_requiring_lock = {}, std::vector const & tables_no_lock = {}) override; - nano::read_transaction tx_begin_read () const override; - - std::string vendor_get () const override; - - void serialize_mdb_tracker (boost::property_tree::ptree &, std::chrono::milliseconds, std::chrono::milliseconds) override; - - static void create_backup_file (nano::mdb_env &, boost::filesystem::path const &, nano::logger_mt &); - - void serialize_memory_stats (boost::property_tree::ptree &) override; - - unsigned max_block_write_batch_num () const override; - - private: - nano::logger_mt & logger; - bool error{ false }; - - public: - nano::mdb_env env; - - bool exists (nano::transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a) const; - - int get (nano::transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a, nano::mdb_val & value_a) const; - int put (nano::write_transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a, nano::mdb_val const & value_a) const; - int del (nano::write_transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a) const; - - bool copy_db (boost::filesystem::path const & destination_file) override; - void rebuild_db (nano::write_transaction const & transaction_a) override; - - template - nano::store_iterator make_iterator (nano::transaction const & transaction_a, tables table_a, bool const direction_asc = true) const - { - return nano::store_iterator (std::make_unique> (transaction_a, table_to_dbi (table_a), nano::mdb_val{}, direction_asc)); - } - - template - nano::store_iterator make_iterator (nano::transaction const & transaction_a, tables table_a, nano::mdb_val const & key) const - { - return nano::store_iterator (std::make_unique> (transaction_a, table_to_dbi (table_a), key)); - } - - bool init_error () const override; - - uint64_t count (nano::transaction const &, MDB_dbi) const; - std::string error_string (int status) const override; - - // These are only use in the upgrade process. - std::shared_ptr block_get_v14 (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_sideband_v14 * sideband_a = nullptr, bool * is_state_v1 = nullptr) const; - std::size_t block_successor_offset_v14 (nano::transaction const & transaction_a, std::size_t entry_size_a, nano::block_type type_a) const; - nano::block_hash block_successor_v14 (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const; - nano::mdb_val block_raw_get_v14 (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type & type_a, bool * is_state_v1 = nullptr) const; - boost::optional block_raw_get_by_type_v14 (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type & type_a, bool * is_state_v1) const; - - private: - bool do_upgrades (nano::write_transaction &, nano::ledger_constants & constants, bool &); - void upgrade_v14_to_v15 (nano::write_transaction &); - void upgrade_v15_to_v16 (nano::write_transaction const &); - void upgrade_v16_to_v17 (nano::write_transaction const &); - void upgrade_v17_to_v18 (nano::write_transaction const &, nano::ledger_constants & constants); - void upgrade_v18_to_v19 (nano::write_transaction const &); - void upgrade_v19_to_v20 (nano::write_transaction const &); - void upgrade_v20_to_v21 (nano::write_transaction const &); - void upgrade_v21_to_v22 (nano::write_transaction const &); - - std::shared_ptr block_get_v18 (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const; - nano::mdb_val block_raw_get_v18 (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type & type_a) const; - boost::optional block_raw_get_by_type_v18 (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type & type_a) const; - nano::uint128_t block_balance_v18 (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const; - - void open_databases (bool &, nano::transaction const &, unsigned); - - int drop (nano::write_transaction const & transaction_a, tables table_a) override; - int clear (nano::write_transaction const & transaction_a, MDB_dbi handle_a); - - bool not_found (int status) const override; - bool success (int status) const override; - void release_assert_success (int const status) const - { - if (!success (status)) - { - release_assert (false, error_string (status)); - } - } - int status_code_not_found () const override; - - MDB_dbi table_to_dbi (tables table_a) const; - - mutable nano::mdb_txn_tracker mdb_txn_tracker; - nano::mdb_txn_callbacks create_txn_callbacks () const; - bool txn_tracking_enabled; - - uint64_t count (nano::transaction const & transaction_a, tables table_a) const override; - - bool vacuum_after_upgrade (boost::filesystem::path const & path_a, nano::lmdb_config const & lmdb_config_a); - - class upgrade_counters - { - public: - upgrade_counters (uint64_t count_before_v0, uint64_t count_before_v1); - bool are_equal () const; - - uint64_t before_v0; - uint64_t before_v1; - uint64_t after_v0{ 0 }; - uint64_t after_v1{ 0 }; - }; - - friend class mdb_block_store_supported_version_upgrades_Test; - friend class mdb_block_store_upgrade_v14_v15_Test; - friend class mdb_block_store_upgrade_v15_v16_Test; - friend class mdb_block_store_upgrade_v16_v17_Test; - friend class mdb_block_store_upgrade_v17_v18_Test; - friend class mdb_block_store_upgrade_v18_v19_Test; - friend class mdb_block_store_upgrade_v19_v20_Test; - friend class mdb_block_store_upgrade_v20_v21_Test; - friend class mdb_block_store_upgrade_v21_v22_Test; - friend class block_store_DISABLED_change_dupsort_Test; - friend void write_sideband_v14 (nano::lmdb::store &, nano::transaction &, nano::block const &, MDB_dbi); - friend void write_sideband_v15 (nano::lmdb::store &, nano::transaction &, nano::block const &); - friend void modify_account_info_to_v14 (nano::lmdb::store &, nano::transaction const &, nano::account const &, uint64_t, nano::block_hash const &); - friend void modify_confirmation_height_to_v15 (nano::lmdb::store &, nano::transaction const &, nano::account const &, uint64_t); - }; -} - -template <> -void * mdb_val::data () const; -template <> -std::size_t mdb_val::size () const; -template <> -mdb_val::db_val (std::size_t size_a, void * data_a); -template <> -void mdb_val::convert_buffer_to_value (); -} diff --git a/nano/node/lmdb/online_weight_store.cpp b/nano/node/lmdb/online_weight_store.cpp deleted file mode 100644 index 17eaa81a64..0000000000 --- a/nano/node/lmdb/online_weight_store.cpp +++ /dev/null @@ -1,45 +0,0 @@ -#include -#include - -nano::lmdb::online_weight_store::online_weight_store (nano::lmdb::store & store_a) : - store{ store_a } -{ -} - -void nano::lmdb::online_weight_store::put (nano::write_transaction const & transaction, uint64_t time, nano::amount const & amount) -{ - auto status = store.put (transaction, tables::online_weight, time, amount); - store.release_assert_success (status); -} - -void nano::lmdb::online_weight_store::del (nano::write_transaction const & transaction, uint64_t time) -{ - auto status = store.del (transaction, tables::online_weight, time); - store.release_assert_success (status); -} - -nano::store_iterator nano::lmdb::online_weight_store::begin (nano::transaction const & transaction) const -{ - return store.make_iterator (transaction, tables::online_weight); -} - -nano::store_iterator nano::lmdb::online_weight_store::rbegin (nano::transaction const & transaction) const -{ - return store.make_iterator (transaction, tables::online_weight, false); -} - -nano::store_iterator nano::lmdb::online_weight_store::end () const -{ - return nano::store_iterator (nullptr); -} - -size_t nano::lmdb::online_weight_store::count (nano::transaction const & transaction) const -{ - return store.count (transaction, tables::online_weight); -} - -void nano::lmdb::online_weight_store::clear (nano::write_transaction const & transaction) -{ - auto status = store.drop (transaction, tables::online_weight); - store.release_assert_success (status); -} diff --git a/nano/node/lmdb/online_weight_store.hpp b/nano/node/lmdb/online_weight_store.hpp deleted file mode 100644 index 88ed78c53c..0000000000 --- a/nano/node/lmdb/online_weight_store.hpp +++ /dev/null @@ -1,33 +0,0 @@ -#pragma once - -#include - -#include - -namespace nano -{ -namespace lmdb -{ - class online_weight_store : public nano::online_weight_store - { - private: - nano::lmdb::store & store; - - public: - explicit online_weight_store (nano::lmdb::store & store_a); - void put (nano::write_transaction const & transaction_a, uint64_t time_a, nano::amount const & amount_a) override; - void del (nano::write_transaction const & transaction_a, uint64_t time_a) override; - nano::store_iterator begin (nano::transaction const & transaction_a) const override; - nano::store_iterator rbegin (nano::transaction const & transaction_a) const override; - nano::store_iterator end () const override; - size_t count (nano::transaction const & transaction_a) const override; - void clear (nano::write_transaction const & transaction_a) override; - - /** - * Samples of online vote weight - * uint64_t -> nano::amount - */ - MDB_dbi online_weight_handle{ 0 }; - }; -} -} diff --git a/nano/node/lmdb/peer_store.cpp b/nano/node/lmdb/peer_store.cpp deleted file mode 100644 index 738bddee85..0000000000 --- a/nano/node/lmdb/peer_store.cpp +++ /dev/null @@ -1,43 +0,0 @@ -#include -#include - -nano::lmdb::peer_store::peer_store (nano::lmdb::store & store) : - store{ store } {}; - -void nano::lmdb::peer_store::put (nano::write_transaction const & transaction, nano::endpoint_key const & endpoint) -{ - auto status = store.put (transaction, tables::peers, endpoint, nullptr); - store.release_assert_success (status); -} - -void nano::lmdb::peer_store::del (nano::write_transaction const & transaction, nano::endpoint_key const & endpoint) -{ - auto status = store.del (transaction, tables::peers, endpoint); - store.release_assert_success (status); -} - -bool nano::lmdb::peer_store::exists (nano::transaction const & transaction, nano::endpoint_key const & endpoint) const -{ - return store.exists (transaction, tables::peers, endpoint); -} - -size_t nano::lmdb::peer_store::count (nano::transaction const & transaction) const -{ - return store.count (transaction, tables::peers); -} - -void nano::lmdb::peer_store::clear (nano::write_transaction const & transaction) -{ - auto status = store.drop (transaction, tables::peers); - store.release_assert_success (status); -} - -nano::store_iterator nano::lmdb::peer_store::begin (nano::transaction const & transaction) const -{ - return store.make_iterator (transaction, tables::peers); -} - -nano::store_iterator nano::lmdb::peer_store::end () const -{ - return nano::store_iterator (nullptr); -} diff --git a/nano/node/lmdb/peer_store.hpp b/nano/node/lmdb/peer_store.hpp deleted file mode 100644 index 018b85e9be..0000000000 --- a/nano/node/lmdb/peer_store.hpp +++ /dev/null @@ -1,34 +0,0 @@ -#pragma once - -#include - -#include - -namespace nano -{ -namespace lmdb -{ - class store; - class peer_store : public nano::peer_store - { - private: - nano::lmdb::store & store; - - public: - explicit peer_store (nano::lmdb::store & store_a); - void put (nano::write_transaction const & transaction_a, nano::endpoint_key const & endpoint_a) override; - void del (nano::write_transaction const & transaction_a, nano::endpoint_key const & endpoint_a) override; - bool exists (nano::transaction const & transaction_a, nano::endpoint_key const & endpoint_a) const override; - size_t count (nano::transaction const & transaction_a) const override; - void clear (nano::write_transaction const & transaction_a) override; - nano::store_iterator begin (nano::transaction const & transaction_a) const override; - nano::store_iterator end () const override; - - /* - * Endpoints for peers - * nano::endpoint_key -> no_value - */ - MDB_dbi peers_handle{ 0 }; - }; -} -} diff --git a/nano/node/lmdb/pending_store.hpp b/nano/node/lmdb/pending_store.hpp deleted file mode 100644 index 30896d65d4..0000000000 --- a/nano/node/lmdb/pending_store.hpp +++ /dev/null @@ -1,48 +0,0 @@ -#pragma once - -#include - -#include - -namespace nano -{ -namespace lmdb -{ - class store; - class pending_store : public nano::pending_store - { - private: - nano::lmdb::store & store; - - public: - explicit pending_store (nano::lmdb::store & store_a); - void put (nano::write_transaction const & transaction_a, nano::pending_key const & key_a, nano::pending_info const & pending_info_a) override; - void del (nano::write_transaction const & transaction_a, nano::pending_key const & key_a) override; - bool get (nano::transaction const & transaction_a, nano::pending_key const & key_a, nano::pending_info & pending_a) override; - bool exists (nano::transaction const & transaction_a, nano::pending_key const & key_a) override; - bool any (nano::transaction const & transaction_a, nano::account const & account_a) override; - nano::store_iterator begin (nano::transaction const & transaction_a, nano::pending_key const & key_a) const override; - nano::store_iterator begin (nano::transaction const & transaction_a) const override; - nano::store_iterator end () const override; - void for_each_par (std::function, nano::store_iterator)> const & action_a) const override; - - /** - * Maps min_version 0 (destination account, pending block) to (source account, amount). (Removed) - * nano::account, nano::block_hash -> nano::account, nano::amount - */ - MDB_dbi pending_v0_handle{ 0 }; - - /** - * Maps min_version 1 (destination account, pending block) to (source account, amount). (Removed) - * nano::account, nano::block_hash -> nano::account, nano::amount - */ - MDB_dbi pending_v1_handle{ 0 }; - - /** - * Maps (destination account, pending block) to (source account, amount, version). (Removed) - * nano::account, nano::block_hash -> nano::account, nano::amount, nano::epoch - */ - MDB_dbi pending_handle{ 0 }; - }; -} -} diff --git a/nano/node/lmdb/pruned_store.cpp b/nano/node/lmdb/pruned_store.cpp deleted file mode 100644 index 60f833ee08..0000000000 --- a/nano/node/lmdb/pruned_store.cpp +++ /dev/null @@ -1,70 +0,0 @@ -#include -#include -#include - -nano::lmdb::pruned_store::pruned_store (nano::lmdb::store & store_a) : - store{ store_a } {}; - -void nano::lmdb::pruned_store::put (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a) -{ - auto status = store.put (transaction_a, tables::pruned, hash_a, nullptr); - store.release_assert_success (status); -} - -void nano::lmdb::pruned_store::del (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a) -{ - auto status = store.del (transaction_a, tables::pruned, hash_a); - store.release_assert_success (status); -} - -bool nano::lmdb::pruned_store::exists (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const -{ - return store.exists (transaction_a, tables::pruned, hash_a); -} - -nano::block_hash nano::lmdb::pruned_store::random (nano::transaction const & transaction) -{ - nano::block_hash random_hash; - nano::random_pool::generate_block (random_hash.bytes.data (), random_hash.bytes.size ()); - auto existing = begin (transaction, random_hash); - if (existing == end ()) - { - existing = begin (transaction); - } - return existing != end () ? existing->first : 0; -} - -size_t nano::lmdb::pruned_store::count (nano::transaction const & transaction_a) const -{ - return store.count (transaction_a, tables::pruned); -} - -void nano::lmdb::pruned_store::clear (nano::write_transaction const & transaction_a) -{ - auto status = store.drop (transaction_a, tables::pruned); - store.release_assert_success (status); -} - -nano::store_iterator nano::lmdb::pruned_store::begin (nano::transaction const & transaction, nano::block_hash const & hash) const -{ - return store.make_iterator (transaction, tables::pruned, hash); -} - -nano::store_iterator nano::lmdb::pruned_store::begin (nano::transaction const & transaction) const -{ - return store.make_iterator (transaction, tables::pruned); -} - -nano::store_iterator nano::lmdb::pruned_store::end () const -{ - return nano::store_iterator (nullptr); -} - -void nano::lmdb::pruned_store::for_each_par (std::function, nano::store_iterator)> const & action_a) const -{ - parallel_traversal ( - [&action_a, this] (nano::uint256_t const & start, nano::uint256_t const & end, bool const is_last) { - auto transaction (this->store.tx_begin_read ()); - action_a (transaction, this->begin (transaction, start), !is_last ? this->begin (transaction, end) : this->end ()); - }); -} diff --git a/nano/node/lmdb/pruned_store.hpp b/nano/node/lmdb/pruned_store.hpp deleted file mode 100644 index 859d017d1c..0000000000 --- a/nano/node/lmdb/pruned_store.hpp +++ /dev/null @@ -1,37 +0,0 @@ -#pragma once - -#include - -#include - -namespace nano -{ -namespace lmdb -{ - class store; - class pruned_store : public nano::pruned_store - { - private: - nano::lmdb::store & store; - - public: - explicit pruned_store (nano::lmdb::store & store_a); - void put (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a) override; - void del (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a) override; - bool exists (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const override; - nano::block_hash random (nano::transaction const & transaction_a) override; - size_t count (nano::transaction const & transaction_a) const override; - void clear (nano::write_transaction const & transaction_a) override; - nano::store_iterator begin (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const override; - nano::store_iterator begin (nano::transaction const & transaction_a) const override; - nano::store_iterator end () const override; - void for_each_par (std::function, nano::store_iterator)> const & action_a) const override; - - /** - * Pruned blocks hashes - * nano::block_hash -> none - */ - MDB_dbi pruned_handle{ 0 }; - }; -} -} diff --git a/nano/node/lmdb/version_store.hpp b/nano/node/lmdb/version_store.hpp deleted file mode 100644 index 9f4a6909f6..0000000000 --- a/nano/node/lmdb/version_store.hpp +++ /dev/null @@ -1,21 +0,0 @@ -#pragma once - -#include - -namespace nano -{ -namespace lmdb -{ - class store; - class version_store : public nano::version_store - { - protected: - nano::lmdb::store & store; - - public: - explicit version_store (nano::lmdb::store & store_a); - void put (nano::write_transaction const & transaction_a, int version_a) override; - int get (nano::transaction const & transaction_a) const override; - }; -} -} diff --git a/nano/node/logging.cpp b/nano/node/logging.cpp index 6a414f0b8b..5b3427963e 100644 --- a/nano/node/logging.cpp +++ b/nano/node/logging.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include diff --git a/nano/node/make_store.cpp b/nano/node/make_store.cpp new file mode 100644 index 0000000000..55b7fa83d2 --- /dev/null +++ b/nano/node/make_store.cpp @@ -0,0 +1,15 @@ +#include +#include +#include + +#include + +std::unique_ptr nano::make_store (nano::logger_mt & logger, boost::filesystem::path const & path, nano::ledger_constants & constants, bool read_only, bool add_db_postfix, nano::rocksdb_config const & rocksdb_config, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a, nano::lmdb_config const & lmdb_config_a, bool backup_before_upgrade) +{ + if (rocksdb_config.enable) + { + return std::make_unique (logger, add_db_postfix ? path / "rocksdb" : path, constants, rocksdb_config, read_only); + } + + return std::make_unique (logger, add_db_postfix ? path / "data.ldb" : path, constants, txn_tracking_config_a, block_processor_batch_max_time_a, lmdb_config_a, backup_before_upgrade); +} diff --git a/nano/node/make_store.hpp b/nano/node/make_store.hpp new file mode 100644 index 0000000000..c0521deba5 --- /dev/null +++ b/nano/node/make_store.hpp @@ -0,0 +1,31 @@ +#pragma once + +#include +#include +#include +#include + +#include + +namespace boost::filesystem +{ +class path; +} + +namespace nano +{ +class ledger_constants; +class lmdb_config; +class rocksdb_config; +class txn_tracking_config; +} + +namespace nano::store +{ +class component; +} + +namespace nano +{ +std::unique_ptr make_store (nano::logger_mt & logger, boost::filesystem::path const & path, nano::ledger_constants & constants, bool open_read_only = false, bool add_db_postfix = true, nano::rocksdb_config const & rocksdb_config = nano::rocksdb_config{}, nano::txn_tracking_config const & txn_tracking_config_a = nano::txn_tracking_config{}, std::chrono::milliseconds block_processor_batch_max_time_a = std::chrono::milliseconds (5000), nano::lmdb_config const & lmdb_config_a = nano::lmdb_config{}, bool backup_before_upgrade = false); +} diff --git a/nano/node/node.cpp b/nano/node/node.cpp index 002d9791f1..0c764d9de4 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -4,8 +4,8 @@ #include #include #include +#include #include -#include #include #include #include @@ -13,6 +13,8 @@ #include #include #include +#include +#include #include #include @@ -220,7 +222,7 @@ nano::node::node (boost::asio::io_context & io_ctx_a, boost::filesystem::path co return ledger.weight (rep); }; - backlog.activate_callback.add ([this] (nano::transaction const & transaction, nano::account const & account, nano::account_info const & account_info, nano::confirmation_height_info const & conf_info) { + backlog.activate_callback.add ([this] (store::transaction const & transaction, nano::account const & account, nano::account_info const & account_info, nano::confirmation_height_info const & conf_info) { scheduler.priority.activate (account, transaction); scheduler.optimistic.activate (account, account_info, conf_info); }); @@ -592,7 +594,7 @@ void nano::node::process_active (std::shared_ptr const & incoming) block_processor.add (incoming); } -[[nodiscard]] nano::process_return nano::node::process (nano::write_transaction const & transaction, nano::block & block) +[[nodiscard]] nano::process_return nano::node::process (store::write_transaction const & transaction, nano::block & block) { return ledger.process (transaction, block); } @@ -1298,7 +1300,7 @@ void nano::node::ongoing_online_weight_calculation () ongoing_online_weight_calculation_queue (); } -void nano::node::receive_confirmed (nano::transaction const & block_transaction_a, nano::block_hash const & hash_a, nano::account const & destination_a) +void nano::node::receive_confirmed (store::transaction const & block_transaction_a, nano::block_hash const & hash_a, nano::account const & destination_a) { nano::unique_lock lk{ wallets.mutex }; auto wallets_l = wallets.get_wallets (); @@ -1332,7 +1334,7 @@ void nano::node::receive_confirmed (nano::transaction const & block_transaction_ } } -void nano::node::process_confirmed_data (nano::transaction const & transaction_a, std::shared_ptr const & block_a, nano::block_hash const & hash_a, nano::account & account_a, nano::uint128_t & amount_a, bool & is_state_send_a, bool & is_state_epoch_a, nano::account & pending_account_a) +void nano::node::process_confirmed_data (store::transaction const & transaction_a, std::shared_ptr const & block_a, nano::block_hash const & hash_a, nano::account & account_a, nano::uint128_t & amount_a, bool & is_state_send_a, bool & is_state_epoch_a, nano::account & pending_account_a) { // Faster account calculation account_a = block_a->account (); @@ -1344,7 +1346,7 @@ void nano::node::process_confirmed_data (nano::transaction const & transaction_a auto previous (block_a->previous ()); bool error (false); auto previous_balance (ledger.balance_safe (transaction_a, previous, error)); - auto block_balance (store.block.balance_calculated (block_a)); + auto block_balance = ledger.balance (*block_a); if (hash_a != ledger.constants.genesis->account ()) { if (!error) @@ -1461,7 +1463,7 @@ void nano::node::bootstrap_block (const nano::block_hash & hash) } /** Convenience function to easily return the confirmation height of an account. */ -uint64_t nano::node::get_confirmation_height (nano::transaction const & transaction_a, nano::account & account_a) +uint64_t nano::node::get_confirmation_height (store::transaction const & transaction_a, nano::account & account_a) { nano::confirmation_height_info info; store.confirmation_height.get (transaction_a, account_a, info); @@ -1570,13 +1572,3 @@ nano::node_flags const & nano::inactive_node_flag_defaults () node_flags.disable_tcp_realtime = true; return node_flags; } - -std::unique_ptr nano::make_store (nano::logger_mt & logger, boost::filesystem::path const & path, nano::ledger_constants & constants, bool read_only, bool add_db_postfix, nano::rocksdb_config const & rocksdb_config, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a, nano::lmdb_config const & lmdb_config_a, bool backup_before_upgrade) -{ - if (rocksdb_config.enable) - { - return std::make_unique (logger, add_db_postfix ? path / "rocksdb" : path, constants, rocksdb_config, read_only); - } - - return std::make_unique (logger, add_db_postfix ? path / "data.ldb" : path, constants, txn_tracking_config_a, block_processor_batch_max_time_a, lmdb_config_a, backup_before_upgrade); -} diff --git a/nano/node/node.hpp b/nano/node/node.hpp index a52a04d3d4..870562dc55 100644 --- a/nano/node/node.hpp +++ b/nano/node/node.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -86,8 +87,8 @@ class node final : public std::enable_shared_from_this void stop (); std::shared_ptr shared (); int store_version (); - void receive_confirmed (nano::transaction const & block_transaction_a, nano::block_hash const & hash_a, nano::account const & destination_a); - void process_confirmed_data (nano::transaction const &, std::shared_ptr const &, nano::block_hash const &, nano::account &, nano::uint128_t &, bool &, bool &, nano::account &); + void receive_confirmed (store::transaction const & block_transaction_a, nano::block_hash const & hash_a, nano::account const & destination_a); + void process_confirmed_data (store::transaction const &, std::shared_ptr const &, nano::block_hash const &, nano::account &, nano::uint128_t &, bool &, bool &, nano::account &); void process_confirmed (nano::election_status const &, uint64_t = 0); void process_active (std::shared_ptr const &); std::optional process_local (std::shared_ptr const &); @@ -130,7 +131,7 @@ class node final : public std::enable_shared_from_this bool online () const; bool init_error () const; std::pair get_bootstrap_weights () const; - uint64_t get_confirmation_height (nano::transaction const &, nano::account &); + uint64_t get_confirmation_height (store::transaction const &, nano::account &); /* * Attempts to bootstrap block. This is the best effort, there is no guarantee that the block will be bootstrapped. */ @@ -151,8 +152,8 @@ class node final : public std::enable_shared_from_this nano::work_pool & work; nano::distributed_work_factory distributed_work; nano::logger_mt logger; - std::unique_ptr store_impl; - nano::store & store; + std::unique_ptr store_impl; + nano::store::component & store; nano::unchecked_map unchecked; std::unique_ptr wallets_store_impl; nano::wallets_store & wallets_store; @@ -221,7 +222,7 @@ class node final : public std::enable_shared_from_this Transaction is comitted before function return */ [[nodiscard]] nano::process_return process (nano::block & block); - [[nodiscard]] nano::process_return process (nano::write_transaction const &, nano::block & block); + [[nodiscard]] nano::process_return process (store::write_transaction const &, nano::block & block); nano::block_hash latest (nano::account const &); nano::uint128_t balance (nano::account const &); diff --git a/nano/node/online_reps.cpp b/nano/node/online_reps.cpp index bf3fe7898d..3fdd2442dd 100644 --- a/nano/node/online_reps.cpp +++ b/nano/node/online_reps.cpp @@ -1,7 +1,8 @@ #include #include #include -#include +#include +#include nano::online_reps::online_reps (nano::ledger & ledger_a, nano::node_config const & config_a) : ledger{ ledger_a }, @@ -64,7 +65,7 @@ nano::uint128_t nano::online_reps::calculate_online () const return current; } -nano::uint128_t nano::online_reps::calculate_trend (nano::transaction & transaction_a) const +nano::uint128_t nano::online_reps::calculate_trend (store::transaction & transaction_a) const { std::vector items; items.reserve (config.network_params.node.max_weight_samples + 1); diff --git a/nano/node/online_reps.hpp b/nano/node/online_reps.hpp index add5328a03..98fd8c519e 100644 --- a/nano/node/online_reps.hpp +++ b/nano/node/online_reps.hpp @@ -16,7 +16,6 @@ namespace nano { class ledger; class node_config; -class transaction; /** Track online representatives and trend online weight */ class online_reps final @@ -51,7 +50,7 @@ class online_reps final class tag_account { }; - nano::uint128_t calculate_trend (nano::transaction &) const; + nano::uint128_t calculate_trend (store::transaction &) const; nano::uint128_t calculate_online () const; mutable nano::mutex mutex; nano::ledger & ledger; diff --git a/nano/node/process_live_dispatcher.cpp b/nano/node/process_live_dispatcher.cpp index 2b231a5258..fa23af6370 100644 --- a/nano/node/process_live_dispatcher.cpp +++ b/nano/node/process_live_dispatcher.cpp @@ -6,7 +6,7 @@ #include #include #include -#include +#include nano::process_live_dispatcher::process_live_dispatcher (nano::ledger & ledger, nano::scheduler::priority & scheduler, nano::vote_cache & inactive_vote_cache, nano::websocket_server & websocket) : ledger{ ledger }, @@ -28,7 +28,7 @@ void nano::process_live_dispatcher::connect (nano::block_processor & block_proce }); } -void nano::process_live_dispatcher::inspect (nano::process_return const & result, nano::block const & block, nano::transaction const & transaction) +void nano::process_live_dispatcher::inspect (nano::process_return const & result, nano::block const & block, store::transaction const & transaction) { switch (result.code) { @@ -40,7 +40,7 @@ void nano::process_live_dispatcher::inspect (nano::process_return const & result } } -void nano::process_live_dispatcher::process_live (nano::block const & block, nano::transaction const & transaction) +void nano::process_live_dispatcher::process_live (nano::block const & block, store::transaction const & transaction) { // Start collecting quorum on block if (ledger.dependents_confirmed (transaction, block)) diff --git a/nano/node/process_live_dispatcher.hpp b/nano/node/process_live_dispatcher.hpp index 23206544d1..c07525a0ff 100644 --- a/nano/node/process_live_dispatcher.hpp +++ b/nano/node/process_live_dispatcher.hpp @@ -1,5 +1,10 @@ #pragma once +namespace nano::store +{ +class transaction; +} + namespace nano { class ledger; @@ -8,7 +13,6 @@ class websocket_server; class block_processor; class process_return; class block; -class transaction; namespace scheduler { @@ -24,8 +28,8 @@ class process_live_dispatcher private: // Block_processor observer - void inspect (nano::process_return const & result, nano::block const & block, nano::transaction const & transaction); - void process_live (nano::block const & block, nano::transaction const & transaction); + void inspect (nano::process_return const & result, nano::block const & block, store::transaction const & transaction); + void process_live (nano::block const & block, store::transaction const & transaction); nano::ledger & ledger; nano::scheduler::priority & scheduler; diff --git a/nano/node/request_aggregator.cpp b/nano/node/request_aggregator.cpp index 59acea1858..cb84ba6045 100644 --- a/nano/node/request_aggregator.cpp +++ b/nano/node/request_aggregator.cpp @@ -8,7 +8,7 @@ #include #include #include -#include +#include nano::request_aggregator::request_aggregator (nano::node_config const & config_a, nano::stats & stats_a, nano::vote_generator & generator_a, nano::vote_generator & final_generator_a, nano::local_vote_history & history_a, nano::ledger & ledger_a, nano::wallets & wallets_a, nano::active_transactions & active_a) : config{ config_a }, diff --git a/nano/node/rocksdb/account_store.cpp b/nano/node/rocksdb/account_store.cpp deleted file mode 100644 index 8cbb5b8ab9..0000000000 --- a/nano/node/rocksdb/account_store.cpp +++ /dev/null @@ -1,72 +0,0 @@ -#include -#include -#include - -nano::rocksdb::account_store::account_store (nano::rocksdb::store & store_a) : - store (store_a){}; - -void nano::rocksdb::account_store::put (nano::write_transaction const & transaction, nano::account const & account, nano::account_info const & info) -{ - auto status = store.put (transaction, tables::accounts, account, info); - store.release_assert_success (status); -} - -bool nano::rocksdb::account_store::get (nano::transaction const & transaction, nano::account const & account, nano::account_info & info) -{ - nano::rocksdb_val value; - auto status1 (store.get (transaction, tables::accounts, account, value)); - release_assert (store.success (status1) || store.not_found (status1)); - bool result (true); - if (store.success (status1)) - { - nano::bufferstream stream (reinterpret_cast (value.data ()), value.size ()); - result = info.deserialize (stream); - } - return result; -} - -void nano::rocksdb::account_store::del (nano::write_transaction const & transaction_a, nano::account const & account_a) -{ - auto status = store.del (transaction_a, tables::accounts, account_a); - store.release_assert_success (status); -} - -bool nano::rocksdb::account_store::exists (nano::transaction const & transaction_a, nano::account const & account_a) -{ - auto iterator (begin (transaction_a, account_a)); - return iterator != end () && nano::account (iterator->first) == account_a; -} - -size_t nano::rocksdb::account_store::count (nano::transaction const & transaction_a) -{ - return store.count (transaction_a, tables::accounts); -} - -nano::store_iterator nano::rocksdb::account_store::begin (nano::transaction const & transaction, nano::account const & account) const -{ - return store.make_iterator (transaction, tables::accounts, account); -} - -nano::store_iterator nano::rocksdb::account_store::begin (nano::transaction const & transaction) const -{ - return store.make_iterator (transaction, tables::accounts); -} - -nano::store_iterator nano::rocksdb::account_store::rbegin (nano::transaction const & transaction_a) const -{ - return store.make_iterator (transaction_a, tables::accounts, false); -} - -nano::store_iterator nano::rocksdb::account_store::end () const -{ - return nano::store_iterator (nullptr); -} - -void nano::rocksdb::account_store::for_each_par (std::function, nano::store_iterator)> const & action_a) const -{ - parallel_traversal ( - [&action_a, this] (nano::uint256_t const & start, nano::uint256_t const & end, bool const is_last) { - auto transaction (this->store.tx_begin_read ()); - action_a (transaction, this->begin (transaction, start), !is_last ? this->begin (transaction, end) : this->end ()); - }); -} diff --git a/nano/node/rocksdb/account_store.hpp b/nano/node/rocksdb/account_store.hpp deleted file mode 100644 index b8ea152775..0000000000 --- a/nano/node/rocksdb/account_store.hpp +++ /dev/null @@ -1,29 +0,0 @@ -#pragma once - -#include - -namespace nano -{ -namespace rocksdb -{ - class store; - class account_store : public nano::account_store - { - private: - nano::rocksdb::store & store; - - public: - explicit account_store (nano::rocksdb::store & store_a); - void put (nano::write_transaction const & transaction, nano::account const & account, nano::account_info const & info) override; - bool get (nano::transaction const & transaction_a, nano::account const & account_a, nano::account_info & info_a) override; - void del (nano::write_transaction const & transaction_a, nano::account const & account_a) override; - bool exists (nano::transaction const & transaction_a, nano::account const & account_a) override; - size_t count (nano::transaction const & transaction_a) override; - nano::store_iterator begin (nano::transaction const & transaction_a, nano::account const & account_a) const override; - nano::store_iterator begin (nano::transaction const & transaction_a) const override; - nano::store_iterator rbegin (nano::transaction const & transaction_a) const override; - nano::store_iterator end () const override; - void for_each_par (std::function, nano::store_iterator)> const & action_a) const override; - }; -} -} diff --git a/nano/node/rocksdb/block_store.cpp b/nano/node/rocksdb/block_store.cpp deleted file mode 100644 index 518b6720fb..0000000000 --- a/nano/node/rocksdb/block_store.cpp +++ /dev/null @@ -1,297 +0,0 @@ -#include -#include -#include - -namespace nano -{ -/** - * Fill in our predecessors - */ -class block_predecessor_rocksdb_set : public nano::block_visitor -{ -public: - block_predecessor_rocksdb_set (nano::write_transaction const & transaction_a, nano::rocksdb::block_store & block_store_a); - virtual ~block_predecessor_rocksdb_set () = default; - void fill_value (nano::block const & block_a); - void send_block (nano::send_block const & block_a) override; - void receive_block (nano::receive_block const & block_a) override; - void open_block (nano::open_block const & block_a) override; - void change_block (nano::change_block const & block_a) override; - void state_block (nano::state_block const & block_a) override; - nano::write_transaction const & transaction; - nano::rocksdb::block_store & block_store; -}; -} - -nano::rocksdb::block_store::block_store (nano::rocksdb::store & store_a) : - store{ store_a } {}; - -void nano::rocksdb::block_store::put (nano::write_transaction const & transaction, nano::block_hash const & hash, nano::block const & block) -{ - debug_assert (block.sideband ().successor.is_zero () || exists (transaction, block.sideband ().successor)); - std::vector vector; - { - nano::vectorstream stream (vector); - nano::serialize_block (stream, block); - block.sideband ().serialize (stream, block.type ()); - } - raw_put (transaction, vector, hash); - block_predecessor_rocksdb_set predecessor (transaction, *this); - block.visit (predecessor); - debug_assert (block.previous ().is_zero () || successor (transaction, block.previous ()) == hash); -} - -void nano::rocksdb::block_store::raw_put (nano::write_transaction const & transaction_a, std::vector const & data, nano::block_hash const & hash_a) -{ - nano::rocksdb_val value{ data.size (), (void *)data.data () }; - auto status = store.put (transaction_a, tables::blocks, hash_a, value); - store.release_assert_success (status); -} - -nano::block_hash nano::rocksdb::block_store::successor (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const -{ - nano::rocksdb_val value; - block_raw_get (transaction_a, hash_a, value); - nano::block_hash result; - if (value.size () != 0) - { - debug_assert (value.size () >= result.bytes.size ()); - auto type = block_type_from_raw (value.data ()); - nano::bufferstream stream (reinterpret_cast (value.data ()) + block_successor_offset (transaction_a, value.size (), type), result.bytes.size ()); - auto error (nano::try_read (stream, result.bytes)); - (void)error; - debug_assert (!error); - } - else - { - result.clear (); - } - return result; -} - -void nano::rocksdb::block_store::successor_clear (nano::write_transaction const & transaction, nano::block_hash const & hash) -{ - nano::rocksdb_val value; - block_raw_get (transaction, hash, value); - debug_assert (value.size () != 0); - auto type = block_type_from_raw (value.data ()); - std::vector data (static_cast (value.data ()), static_cast (value.data ()) + value.size ()); - std::fill_n (data.begin () + block_successor_offset (transaction, value.size (), type), sizeof (nano::block_hash), uint8_t{ 0 }); - raw_put (transaction, data, hash); -} - -std::shared_ptr nano::rocksdb::block_store::get (nano::transaction const & transaction, nano::block_hash const & hash) const -{ - nano::rocksdb_val value; - block_raw_get (transaction, hash, value); - std::shared_ptr result; - if (value.size () != 0) - { - nano::bufferstream stream (reinterpret_cast (value.data ()), value.size ()); - nano::block_type type; - auto error (try_read (stream, type)); - release_assert (!error); - result = nano::deserialize_block (stream, type); - release_assert (result != nullptr); - nano::block_sideband sideband; - error = (sideband.deserialize (stream, type)); - release_assert (!error); - result->sideband_set (sideband); - } - return result; -} - -std::shared_ptr nano::rocksdb::block_store::get_no_sideband (nano::transaction const & transaction, nano::block_hash const & hash) const -{ - nano::rocksdb_val value; - block_raw_get (transaction, hash, value); - std::shared_ptr result; - if (value.size () != 0) - { - nano::bufferstream stream (reinterpret_cast (value.data ()), value.size ()); - result = nano::deserialize_block (stream); - debug_assert (result != nullptr); - } - return result; -} - -std::shared_ptr nano::rocksdb::block_store::random (nano::transaction const & transaction) -{ - nano::block_hash hash; - nano::random_pool::generate_block (hash.bytes.data (), hash.bytes.size ()); - auto existing = begin (transaction, hash); - if (existing == end ()) - { - existing = begin (transaction); - } - debug_assert (existing != end ()); - return existing->second.block; -} - -void nano::rocksdb::block_store::del (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a) -{ - auto status = store.del (transaction_a, tables::blocks, hash_a); - store.release_assert_success (status); -} - -bool nano::rocksdb::block_store::exists (nano::transaction const & transaction, nano::block_hash const & hash) -{ - nano::rocksdb_val junk; - block_raw_get (transaction, hash, junk); - return junk.size () != 0; -} - -uint64_t nano::rocksdb::block_store::count (nano::transaction const & transaction_a) -{ - return store.count (transaction_a, tables::blocks); -} - -nano::account nano::rocksdb::block_store::account (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const -{ - auto block (get (transaction_a, hash_a)); - debug_assert (block != nullptr); - return account_calculated (*block); -} - -nano::account nano::rocksdb::block_store::account_calculated (nano::block const & block_a) const -{ - debug_assert (block_a.has_sideband ()); - nano::account result (block_a.account ()); - if (result.is_zero ()) - { - result = block_a.sideband ().account; - } - debug_assert (!result.is_zero ()); - return result; -} - -nano::store_iterator nano::rocksdb::block_store::begin (nano::transaction const & transaction) const -{ - return store.make_iterator (transaction, tables::blocks); -} - -nano::store_iterator nano::rocksdb::block_store::begin (nano::transaction const & transaction, nano::block_hash const & hash) const -{ - return store.make_iterator (transaction, tables::blocks, hash); -} - -nano::store_iterator nano::rocksdb::block_store::end () const -{ - return nano::store_iterator (nullptr); -} - -nano::uint128_t nano::rocksdb::block_store::balance (nano::transaction const & transaction_a, nano::block_hash const & hash_a) -{ - auto block (get (transaction_a, hash_a)); - release_assert (block); - nano::uint128_t result (balance_calculated (block)); - return result; -} - -nano::uint128_t nano::rocksdb::block_store::balance_calculated (std::shared_ptr const & block_a) const -{ - nano::uint128_t result; - switch (block_a->type ()) - { - case nano::block_type::open: - case nano::block_type::receive: - case nano::block_type::change: - result = block_a->sideband ().balance.number (); - break; - case nano::block_type::send: - result = boost::polymorphic_downcast (block_a.get ())->hashables.balance.number (); - break; - case nano::block_type::state: - result = boost::polymorphic_downcast (block_a.get ())->hashables.balance.number (); - break; - case nano::block_type::invalid: - case nano::block_type::not_a_block: - release_assert (false); - break; - } - return result; -} - -nano::epoch nano::rocksdb::block_store::version (nano::transaction const & transaction_a, nano::block_hash const & hash_a) -{ - auto block = get (transaction_a, hash_a); - if (block && block->type () == nano::block_type::state) - { - return block->sideband ().details.epoch; - } - - return nano::epoch::epoch_0; -} - -void nano::rocksdb::block_store::for_each_par (std::function, nano::store_iterator)> const & action_a) const -{ - parallel_traversal ( - [&action_a, this] (nano::uint256_t const & start, nano::uint256_t const & end, bool const is_last) { - auto transaction (this->store.tx_begin_read ()); - action_a (transaction, this->begin (transaction, start), !is_last ? this->begin (transaction, end) : this->end ()); - }); -} - -// Converts a block hash to a block height -uint64_t nano::rocksdb::block_store::account_height (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const -{ - auto block = get (transaction_a, hash_a); - return block->sideband ().height; -} - -void nano::rocksdb::block_store::block_raw_get (nano::transaction const & transaction, nano::block_hash const & hash, nano::rocksdb_val & value) const -{ - auto status = store.get (transaction, tables::blocks, hash, value); - release_assert (store.success (status) || store.not_found (status)); -} - -size_t nano::rocksdb::block_store::block_successor_offset (nano::transaction const & transaction_a, size_t entry_size_a, nano::block_type type_a) const -{ - return entry_size_a - nano::block_sideband::size (type_a); -} - -nano::block_type nano::rocksdb::block_store::block_type_from_raw (void * data_a) -{ - // The block type is the first byte - return static_cast ((reinterpret_cast (data_a))[0]); -} - -nano::block_predecessor_rocksdb_set::block_predecessor_rocksdb_set (nano::write_transaction const & transaction_a, nano::rocksdb::block_store & block_store_a) : - transaction{ transaction_a }, - block_store{ block_store_a } -{ -} -void nano::block_predecessor_rocksdb_set::fill_value (nano::block const & block_a) -{ - auto hash = block_a.hash (); - nano::rocksdb_val value; - block_store.block_raw_get (transaction, block_a.previous (), value); - debug_assert (value.size () != 0); - auto type = block_store.block_type_from_raw (value.data ()); - std::vector data (static_cast (value.data ()), static_cast (value.data ()) + value.size ()); - std::copy (hash.bytes.begin (), hash.bytes.end (), data.begin () + block_store.block_successor_offset (transaction, value.size (), type)); - block_store.raw_put (transaction, data, block_a.previous ()); -} -void nano::block_predecessor_rocksdb_set::send_block (nano::send_block const & block_a) -{ - fill_value (block_a); -} -void nano::block_predecessor_rocksdb_set::receive_block (nano::receive_block const & block_a) -{ - fill_value (block_a); -} -void nano::block_predecessor_rocksdb_set::open_block (nano::open_block const & block_a) -{ - // Open blocks don't have a predecessor -} -void nano::block_predecessor_rocksdb_set::change_block (nano::change_block const & block_a) -{ - fill_value (block_a); -} -void nano::block_predecessor_rocksdb_set::state_block (nano::state_block const & block_a) -{ - if (!block_a.previous ().is_zero ()) - { - fill_value (block_a); - } -} diff --git a/nano/node/rocksdb/block_store.hpp b/nano/node/rocksdb/block_store.hpp deleted file mode 100644 index fc8b07c523..0000000000 --- a/nano/node/rocksdb/block_store.hpp +++ /dev/null @@ -1,49 +0,0 @@ -#pragma once - -#include - -#include - -namespace nano -{ -using rocksdb_val = db_val<::rocksdb::Slice>; -class block_predecessor_rocksdb_set; -namespace rocksdb -{ - class store; - class block_store : public nano::block_store - { - friend class nano::block_predecessor_rocksdb_set; - nano::rocksdb::store & store; - - public: - explicit block_store (nano::rocksdb::store & store_a); - void put (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a, nano::block const & block_a) override; - void raw_put (nano::write_transaction const & transaction_a, std::vector const & data, nano::block_hash const & hash_a) override; - nano::block_hash successor (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const override; - void successor_clear (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a) override; - std::shared_ptr get (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const override; - std::shared_ptr get_no_sideband (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const override; - std::shared_ptr random (nano::transaction const & transaction_a) override; - void del (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a) override; - bool exists (nano::transaction const & transaction_a, nano::block_hash const & hash_a) override; - uint64_t count (nano::transaction const & transaction_a) override; - nano::account account (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const override; - nano::account account_calculated (nano::block const & block_a) const override; - nano::store_iterator begin (nano::transaction const & transaction_a) const override; - nano::store_iterator begin (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const override; - nano::store_iterator end () const override; - nano::uint128_t balance (nano::transaction const & transaction_a, nano::block_hash const & hash_a) override; - nano::uint128_t balance_calculated (std::shared_ptr const & block_a) const override; - nano::epoch version (nano::transaction const & transaction_a, nano::block_hash const & hash_a) override; - void for_each_par (std::function, nano::store_iterator)> const & action_a) const override; - // Converts a block hash to a block height - uint64_t account_height (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const override; - - protected: - void block_raw_get (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::rocksdb_val & value) const; - size_t block_successor_offset (nano::transaction const & transaction_a, size_t entry_size_a, nano::block_type type_a) const; - static nano::block_type block_type_from_raw (void * data_a); - }; -} -} diff --git a/nano/node/rocksdb/confirmation_height_store.cpp b/nano/node/rocksdb/confirmation_height_store.cpp deleted file mode 100644 index 8db8a6999e..0000000000 --- a/nano/node/rocksdb/confirmation_height_store.cpp +++ /dev/null @@ -1,84 +0,0 @@ -#include -#include -#include - -nano::rocksdb::confirmation_height_store::confirmation_height_store (nano::rocksdb::store & store) : - store{ store } -{ -} - -void nano::rocksdb::confirmation_height_store::put (nano::write_transaction const & transaction, nano::account const & account, nano::confirmation_height_info const & confirmation_height_info) -{ - auto status = store.put (transaction, tables::confirmation_height, account, confirmation_height_info); - store.release_assert_success (status); -} - -bool nano::rocksdb::confirmation_height_store::get (nano::transaction const & transaction, nano::account const & account, nano::confirmation_height_info & confirmation_height_info) -{ - nano::rocksdb_val value; - auto status = store.get (transaction, tables::confirmation_height, account, value); - release_assert (store.success (status) || store.not_found (status)); - bool result (true); - if (store.success (status)) - { - nano::bufferstream stream (reinterpret_cast (value.data ()), value.size ()); - result = confirmation_height_info.deserialize (stream); - } - if (result) - { - confirmation_height_info.height = 0; - confirmation_height_info.frontier = 0; - } - - return result; -} - -bool nano::rocksdb::confirmation_height_store::exists (nano::transaction const & transaction, nano::account const & account) const -{ - return store.exists (transaction, tables::confirmation_height, account); -} - -void nano::rocksdb::confirmation_height_store::del (nano::write_transaction const & transaction, nano::account const & account) -{ - auto status = store.del (transaction, tables::confirmation_height, account); - store.release_assert_success (status); -} - -uint64_t nano::rocksdb::confirmation_height_store::count (nano::transaction const & transaction) -{ - return store.count (transaction, tables::confirmation_height); -} - -void nano::rocksdb::confirmation_height_store::clear (nano::write_transaction const & transaction, nano::account const & account) -{ - del (transaction, account); -} - -void nano::rocksdb::confirmation_height_store::clear (nano::write_transaction const & transaction) -{ - store.drop (transaction, nano::tables::confirmation_height); -} - -nano::store_iterator nano::rocksdb::confirmation_height_store::begin (nano::transaction const & transaction, nano::account const & account) const -{ - return store.make_iterator (transaction, tables::confirmation_height, account); -} - -nano::store_iterator nano::rocksdb::confirmation_height_store::begin (nano::transaction const & transaction) const -{ - return store.make_iterator (transaction, tables::confirmation_height); -} - -nano::store_iterator nano::rocksdb::confirmation_height_store::end () const -{ - return nano::store_iterator (nullptr); -} - -void nano::rocksdb::confirmation_height_store::for_each_par (std::function, nano::store_iterator)> const & action_a) const -{ - parallel_traversal ( - [&action_a, this] (nano::uint256_t const & start, nano::uint256_t const & end, bool const is_last) { - auto transaction (this->store.tx_begin_read ()); - action_a (transaction, this->begin (transaction, start), !is_last ? this->begin (transaction, end) : this->end ()); - }); -} diff --git a/nano/node/rocksdb/confirmation_height_store.hpp b/nano/node/rocksdb/confirmation_height_store.hpp deleted file mode 100644 index c7787c1fa6..0000000000 --- a/nano/node/rocksdb/confirmation_height_store.hpp +++ /dev/null @@ -1,29 +0,0 @@ -#pragma once - -#include - -namespace nano -{ -namespace rocksdb -{ - class store; - class confirmation_height_store : public nano::confirmation_height_store - { - nano::rocksdb::store & store; - - public: - explicit confirmation_height_store (nano::rocksdb::store & store_a); - void put (nano::write_transaction const & transaction_a, nano::account const & account_a, nano::confirmation_height_info const & confirmation_height_info_a) override; - bool get (nano::transaction const & transaction_a, nano::account const & account_a, nano::confirmation_height_info & confirmation_height_info_a) override; - bool exists (nano::transaction const & transaction_a, nano::account const & account_a) const override; - void del (nano::write_transaction const & transaction_a, nano::account const & account_a) override; - uint64_t count (nano::transaction const & transaction_a) override; - void clear (nano::write_transaction const & transaction_a, nano::account const & account_a) override; - void clear (nano::write_transaction const & transaction_a) override; - nano::store_iterator begin (nano::transaction const & transaction_a, nano::account const & account_a) const override; - nano::store_iterator begin (nano::transaction const & transaction_a) const override; - nano::store_iterator end () const override; - void for_each_par (std::function, nano::store_iterator)> const & action_a) const override; - }; -} -} diff --git a/nano/node/rocksdb/final_vote_store.hpp b/nano/node/rocksdb/final_vote_store.hpp deleted file mode 100644 index 71c7e31553..0000000000 --- a/nano/node/rocksdb/final_vote_store.hpp +++ /dev/null @@ -1,29 +0,0 @@ -#pragma once - -#include - -namespace nano -{ -namespace rocksdb -{ - class store; - class final_vote_store : public nano::final_vote_store - { - private: - nano::rocksdb::store & store; - - public: - explicit final_vote_store (nano::rocksdb::store & store); - bool put (nano::write_transaction const & transaction_a, nano::qualified_root const & root_a, nano::block_hash const & hash_a) override; - std::vector get (nano::transaction const & transaction_a, nano::root const & root_a) override; - void del (nano::write_transaction const & transaction_a, nano::root const & root_a) override; - size_t count (nano::transaction const & transaction_a) const override; - void clear (nano::write_transaction const & transaction_a, nano::root const & root_a) override; - void clear (nano::write_transaction const & transaction_a) override; - nano::store_iterator begin (nano::transaction const & transaction_a, nano::qualified_root const & root_a) const override; - nano::store_iterator begin (nano::transaction const & transaction_a) const override; - nano::store_iterator end () const override; - void for_each_par (std::function, nano::store_iterator)> const & action_a) const override; - }; -} -} diff --git a/nano/node/rocksdb/frontier_store.cpp b/nano/node/rocksdb/frontier_store.cpp deleted file mode 100644 index ffc71290c6..0000000000 --- a/nano/node/rocksdb/frontier_store.cpp +++ /dev/null @@ -1,57 +0,0 @@ -#include -#include -#include - -nano::rocksdb::frontier_store::frontier_store (nano::rocksdb::store & store) : - store{ store } -{ -} - -void nano::rocksdb::frontier_store::put (nano::write_transaction const & transaction, nano::block_hash const & block, nano::account const & account) -{ - auto status = store.put (transaction, tables::frontiers, block, account); - store.release_assert_success (status); -} - -nano::account nano::rocksdb::frontier_store::get (nano::transaction const & transaction, nano::block_hash const & hash) const -{ - nano::db_val<::rocksdb::Slice> value; - auto status = store.get (transaction, tables::frontiers, hash, value); - release_assert (store.success (status) || store.not_found (status)); - nano::account result{}; - if (store.success (status)) - { - result = static_cast (value); - } - return result; -} - -void nano::rocksdb::frontier_store::del (nano::write_transaction const & transaction, nano::block_hash const & hash) -{ - auto status = store.del (transaction, tables::frontiers, hash); - store.release_assert_success (status); -} - -nano::store_iterator nano::rocksdb::frontier_store::begin (nano::transaction const & transaction) const -{ - return store.make_iterator (transaction, tables::frontiers); -} - -nano::store_iterator nano::rocksdb::frontier_store::begin (nano::transaction const & transaction, nano::block_hash const & hash) const -{ - return store.make_iterator (transaction, tables::frontiers, hash); -} - -nano::store_iterator nano::rocksdb::frontier_store::end () const -{ - return nano::store_iterator (nullptr); -} - -void nano::rocksdb::frontier_store::for_each_par (std::function, nano::store_iterator)> const & action_a) const -{ - parallel_traversal ( - [&action_a, this] (nano::uint256_t const & start, nano::uint256_t const & end, bool const is_last) { - auto transaction (this->store.tx_begin_read ()); - action_a (transaction, this->begin (transaction, start), !is_last ? this->begin (transaction, end) : this->end ()); - }); -} diff --git a/nano/node/rocksdb/frontier_store.hpp b/nano/node/rocksdb/frontier_store.hpp deleted file mode 100644 index 5b3131d396..0000000000 --- a/nano/node/rocksdb/frontier_store.hpp +++ /dev/null @@ -1,26 +0,0 @@ -#pragma once - -#include - -namespace nano -{ -namespace rocksdb -{ - class store; - class frontier_store : public nano::frontier_store - { - public: - frontier_store (nano::rocksdb::store & store); - void put (nano::write_transaction const &, nano::block_hash const &, nano::account const &) override; - nano::account get (nano::transaction const &, nano::block_hash const &) const override; - void del (nano::write_transaction const &, nano::block_hash const &) override; - nano::store_iterator begin (nano::transaction const &) const override; - nano::store_iterator begin (nano::transaction const &, nano::block_hash const &) const override; - nano::store_iterator end () const override; - void for_each_par (std::function, nano::store_iterator)> const & action_a) const override; - - private: - nano::rocksdb::store & store; - }; -} -} diff --git a/nano/node/rocksdb/online_weight_store.cpp b/nano/node/rocksdb/online_weight_store.cpp deleted file mode 100644 index 43d2b8b9fd..0000000000 --- a/nano/node/rocksdb/online_weight_store.cpp +++ /dev/null @@ -1,45 +0,0 @@ -#include -#include - -nano::rocksdb::online_weight_store::online_weight_store (nano::rocksdb::store & store_a) : - store{ store_a } -{ -} - -void nano::rocksdb::online_weight_store::put (nano::write_transaction const & transaction, uint64_t time, nano::amount const & amount) -{ - auto status = store.put (transaction, tables::online_weight, time, amount); - store.release_assert_success (status); -} - -void nano::rocksdb::online_weight_store::del (nano::write_transaction const & transaction, uint64_t time) -{ - auto status = store.del (transaction, tables::online_weight, time); - store.release_assert_success (status); -} - -nano::store_iterator nano::rocksdb::online_weight_store::begin (nano::transaction const & transaction) const -{ - return store.make_iterator (transaction, tables::online_weight); -} - -nano::store_iterator nano::rocksdb::online_weight_store::rbegin (nano::transaction const & transaction) const -{ - return store.make_iterator (transaction, tables::online_weight, false); -} - -nano::store_iterator nano::rocksdb::online_weight_store::end () const -{ - return nano::store_iterator (nullptr); -} - -size_t nano::rocksdb::online_weight_store::count (nano::transaction const & transaction) const -{ - return store.count (transaction, tables::online_weight); -} - -void nano::rocksdb::online_weight_store::clear (nano::write_transaction const & transaction) -{ - auto status = store.drop (transaction, tables::online_weight); - store.release_assert_success (status); -} diff --git a/nano/node/rocksdb/online_weight_store.hpp b/nano/node/rocksdb/online_weight_store.hpp deleted file mode 100644 index 740518ee5b..0000000000 --- a/nano/node/rocksdb/online_weight_store.hpp +++ /dev/null @@ -1,26 +0,0 @@ -#pragma once - -#include - -namespace nano -{ -namespace rocksdb -{ - class store; - class online_weight_store : public nano::online_weight_store - { - private: - nano::rocksdb::store & store; - - public: - explicit online_weight_store (nano::rocksdb::store & store_a); - void put (nano::write_transaction const & transaction_a, uint64_t time_a, nano::amount const & amount_a) override; - void del (nano::write_transaction const & transaction_a, uint64_t time_a) override; - nano::store_iterator begin (nano::transaction const & transaction_a) const override; - nano::store_iterator rbegin (nano::transaction const & transaction_a) const override; - nano::store_iterator end () const override; - size_t count (nano::transaction const & transaction_a) const override; - void clear (nano::write_transaction const & transaction_a) override; - }; -} -} diff --git a/nano/node/rocksdb/peer_store.cpp b/nano/node/rocksdb/peer_store.cpp deleted file mode 100644 index fd31281f7b..0000000000 --- a/nano/node/rocksdb/peer_store.cpp +++ /dev/null @@ -1,43 +0,0 @@ -#include -#include - -nano::rocksdb::peer_store::peer_store (nano::rocksdb::store & store) : - store{ store } {}; - -void nano::rocksdb::peer_store::put (nano::write_transaction const & transaction, nano::endpoint_key const & endpoint) -{ - auto status = store.put (transaction, tables::peers, endpoint, nullptr); - store.release_assert_success (status); -} - -void nano::rocksdb::peer_store::del (nano::write_transaction const & transaction, nano::endpoint_key const & endpoint) -{ - auto status = store.del (transaction, tables::peers, endpoint); - store.release_assert_success (status); -} - -bool nano::rocksdb::peer_store::exists (nano::transaction const & transaction, nano::endpoint_key const & endpoint) const -{ - return store.exists (transaction, tables::peers, endpoint); -} - -size_t nano::rocksdb::peer_store::count (nano::transaction const & transaction) const -{ - return store.count (transaction, tables::peers); -} - -void nano::rocksdb::peer_store::clear (nano::write_transaction const & transaction) -{ - auto status = store.drop (transaction, tables::peers); - store.release_assert_success (status); -} - -nano::store_iterator nano::rocksdb::peer_store::begin (nano::transaction const & transaction) const -{ - return store.make_iterator (transaction, tables::peers); -} - -nano::store_iterator nano::rocksdb::peer_store::end () const -{ - return nano::store_iterator (nullptr); -} diff --git a/nano/node/rocksdb/peer_store.hpp b/nano/node/rocksdb/peer_store.hpp deleted file mode 100644 index 9d35c3b829..0000000000 --- a/nano/node/rocksdb/peer_store.hpp +++ /dev/null @@ -1,26 +0,0 @@ -#pragma once - -#include - -namespace nano -{ -namespace rocksdb -{ - class store; - class peer_store : public nano::peer_store - { - private: - nano::rocksdb::store & store; - - public: - explicit peer_store (nano::rocksdb::store & store_a); - void put (nano::write_transaction const & transaction_a, nano::endpoint_key const & endpoint_a) override; - void del (nano::write_transaction const & transaction_a, nano::endpoint_key const & endpoint_a) override; - bool exists (nano::transaction const & transaction_a, nano::endpoint_key const & endpoint_a) const override; - size_t count (nano::transaction const & transaction_a) const override; - void clear (nano::write_transaction const & transaction_a) override; - nano::store_iterator begin (nano::transaction const & transaction_a) const override; - nano::store_iterator end () const override; - }; -} -} diff --git a/nano/node/rocksdb/pending_store.hpp b/nano/node/rocksdb/pending_store.hpp deleted file mode 100644 index 8268fa3506..0000000000 --- a/nano/node/rocksdb/pending_store.hpp +++ /dev/null @@ -1,28 +0,0 @@ -#pragma once - -#include - -namespace nano -{ -namespace rocksdb -{ - class store; - class pending_store : public nano::pending_store - { - private: - nano::rocksdb::store & store; - - public: - explicit pending_store (nano::rocksdb::store & store_a); - void put (nano::write_transaction const & transaction_a, nano::pending_key const & key_a, nano::pending_info const & pending_info_a) override; - void del (nano::write_transaction const & transaction_a, nano::pending_key const & key_a) override; - bool get (nano::transaction const & transaction_a, nano::pending_key const & key_a, nano::pending_info & pending_a) override; - bool exists (nano::transaction const & transaction_a, nano::pending_key const & key_a) override; - bool any (nano::transaction const & transaction_a, nano::account const & account_a) override; - nano::store_iterator begin (nano::transaction const & transaction_a, nano::pending_key const & key_a) const override; - nano::store_iterator begin (nano::transaction const & transaction_a) const override; - nano::store_iterator end () const override; - void for_each_par (std::function, nano::store_iterator)> const & action_a) const override; - }; -} -} diff --git a/nano/node/rocksdb/pruned_store.cpp b/nano/node/rocksdb/pruned_store.cpp deleted file mode 100644 index b02423aab9..0000000000 --- a/nano/node/rocksdb/pruned_store.cpp +++ /dev/null @@ -1,70 +0,0 @@ -#include -#include -#include - -nano::rocksdb::pruned_store::pruned_store (nano::rocksdb::store & store_a) : - store{ store_a } {}; - -void nano::rocksdb::pruned_store::put (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a) -{ - auto status = store.put (transaction_a, tables::pruned, hash_a, nullptr); - store.release_assert_success (status); -} - -void nano::rocksdb::pruned_store::del (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a) -{ - auto status = store.del (transaction_a, tables::pruned, hash_a); - store.release_assert_success (status); -} - -bool nano::rocksdb::pruned_store::exists (nano::transaction const & transaction, nano::block_hash const & hash_a) const -{ - return store.exists (transaction, tables::pruned, hash_a); -} - -nano::block_hash nano::rocksdb::pruned_store::random (nano::transaction const & transaction) -{ - nano::block_hash random_hash; - nano::random_pool::generate_block (random_hash.bytes.data (), random_hash.bytes.size ()); - auto existing = begin (transaction, random_hash); - if (existing == end ()) - { - existing = begin (transaction); - } - return existing != end () ? existing->first : 0; -} - -size_t nano::rocksdb::pruned_store::count (nano::transaction const & transaction_a) const -{ - return store.count (transaction_a, tables::pruned); -} - -void nano::rocksdb::pruned_store::clear (nano::write_transaction const & transaction_a) -{ - auto status = store.drop (transaction_a, tables::pruned); - store.release_assert_success (status); -} - -nano::store_iterator nano::rocksdb::pruned_store::begin (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const -{ - return store.make_iterator (transaction_a, tables::pruned, hash_a); -} - -nano::store_iterator nano::rocksdb::pruned_store::begin (nano::transaction const & transaction_a) const -{ - return store.make_iterator (transaction_a, tables::pruned); -} - -nano::store_iterator nano::rocksdb::pruned_store::end () const -{ - return nano::store_iterator (nullptr); -} - -void nano::rocksdb::pruned_store::for_each_par (std::function, nano::store_iterator)> const & action_a) const -{ - parallel_traversal ( - [&action_a, this] (nano::uint256_t const & start, nano::uint256_t const & end, bool const is_last) { - auto transaction (this->store.tx_begin_read ()); - action_a (transaction, this->begin (transaction, start), !is_last ? this->begin (transaction, end) : this->end ()); - }); -} diff --git a/nano/node/rocksdb/pruned_store.hpp b/nano/node/rocksdb/pruned_store.hpp deleted file mode 100644 index 22e1f2e0fe..0000000000 --- a/nano/node/rocksdb/pruned_store.hpp +++ /dev/null @@ -1,29 +0,0 @@ -#pragma once - -#include - -namespace nano -{ -namespace rocksdb -{ - class store; - class pruned_store : public nano::pruned_store - { - private: - nano::rocksdb::store & store; - - public: - explicit pruned_store (nano::rocksdb::store & store_a); - void put (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a) override; - void del (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a) override; - bool exists (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const override; - nano::block_hash random (nano::transaction const & transaction_a) override; - size_t count (nano::transaction const & transaction_a) const override; - void clear (nano::write_transaction const & transaction_a) override; - nano::store_iterator begin (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const override; - nano::store_iterator begin (nano::transaction const & transaction_a) const override; - nano::store_iterator end () const override; - void for_each_par (std::function, nano::store_iterator)> const & action_a) const override; - }; -} -} diff --git a/nano/node/rocksdb/rocksdb.hpp b/nano/node/rocksdb/rocksdb.hpp deleted file mode 100644 index 47f3e71858..0000000000 --- a/nano/node/rocksdb/rocksdb.hpp +++ /dev/null @@ -1,179 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -namespace nano -{ -class logging_mt; -class rocksdb_config; -class rocksdb_block_store_tombstone_count_Test; - -namespace rocksdb -{ - class rocksdb_block_store_upgrade_v21_v22_Test; - - /** - * rocksdb implementation of the block store - */ - class store : public nano::store - { - private: - nano::rocksdb::account_store account_store; - nano::rocksdb::block_store block_store; - nano::rocksdb::confirmation_height_store confirmation_height_store; - nano::rocksdb::final_vote_store final_vote_store; - nano::rocksdb::frontier_store frontier_store; - nano::rocksdb::online_weight_store online_weight_store; - nano::rocksdb::peer_store peer_store; - nano::rocksdb::pending_store pending_store; - nano::rocksdb::pruned_store pruned_store; - nano::rocksdb::version_store version_store; - - public: - friend class nano::rocksdb::account_store; - friend class nano::rocksdb::block_store; - friend class nano::rocksdb::confirmation_height_store; - friend class nano::rocksdb::final_vote_store; - friend class nano::rocksdb::frontier_store; - friend class nano::rocksdb::online_weight_store; - friend class nano::rocksdb::peer_store; - friend class nano::rocksdb::pending_store; - friend class nano::rocksdb::pruned_store; - friend class nano::rocksdb::version_store; - - explicit store (nano::logger_mt &, boost::filesystem::path const &, nano::ledger_constants & constants, nano::rocksdb_config const & = nano::rocksdb_config{}, bool open_read_only = false); - - nano::write_transaction tx_begin_write (std::vector const & tables_requiring_lock = {}, std::vector const & tables_no_lock = {}) override; - nano::read_transaction tx_begin_read () const override; - - std::string vendor_get () const override; - - uint64_t count (nano::transaction const & transaction_a, tables table_a) const override; - - bool exists (nano::transaction const & transaction_a, tables table_a, nano::rocksdb_val const & key_a) const; - int get (nano::transaction const & transaction_a, tables table_a, nano::rocksdb_val const & key_a, nano::rocksdb_val & value_a) const; - int put (nano::write_transaction const & transaction_a, tables table_a, nano::rocksdb_val const & key_a, nano::rocksdb_val const & value_a); - int del (nano::write_transaction const & transaction_a, tables table_a, nano::rocksdb_val const & key_a); - - void serialize_memory_stats (boost::property_tree::ptree &) override; - - bool copy_db (boost::filesystem::path const & destination) override; - void rebuild_db (nano::write_transaction const & transaction_a) override; - - unsigned max_block_write_batch_num () const override; - - template - nano::store_iterator make_iterator (nano::transaction const & transaction_a, tables table_a, bool const direction_asc = true) const - { - return nano::store_iterator (std::make_unique> (db.get (), transaction_a, table_to_column_family (table_a), nullptr, direction_asc)); - } - - template - nano::store_iterator make_iterator (nano::transaction const & transaction_a, tables table_a, nano::rocksdb_val const & key) const - { - return nano::store_iterator (std::make_unique> (db.get (), transaction_a, table_to_column_family (table_a), &key, true)); - } - - bool init_error () const override; - - std::string error_string (int status) const override; - - private: - bool error{ false }; - nano::logger_mt & logger; - nano::ledger_constants & constants; - // Optimistic transactions are used in write mode - ::rocksdb::OptimisticTransactionDB * optimistic_db = nullptr; - std::unique_ptr<::rocksdb::DB> db; - std::vector> handles; - std::shared_ptr<::rocksdb::TableFactory> small_table_factory; - std::unordered_map write_lock_mutexes; - nano::rocksdb_config rocksdb_config; - unsigned const max_block_write_batch_num_m; - - class tombstone_info - { - public: - tombstone_info (uint64_t, uint64_t const); - std::atomic num_since_last_flush; - uint64_t const max; - }; - - std::unordered_map tombstone_map; - std::unordered_map cf_name_table_map; - - ::rocksdb::Transaction * tx (nano::transaction const & transaction_a) const; - std::vector all_tables () const; - - bool not_found (int status) const override; - bool success (int status) const override; - void release_assert_success (int const status) const - { - if (!success (status)) - { - release_assert (false, error_string (status)); - } - } - int status_code_not_found () const override; - int drop (nano::write_transaction const &, tables) override; - - std::vector<::rocksdb::ColumnFamilyDescriptor> get_single_column_family (std::string cf_name) const; - std::vector<::rocksdb::ColumnFamilyDescriptor> get_current_column_families (std::string const & path_a, ::rocksdb::Options const & options_a) const; - ::rocksdb::ColumnFamilyHandle * get_column_family (char const * name) const; - bool column_family_exists (char const * name) const; - ::rocksdb::ColumnFamilyHandle * table_to_column_family (tables table_a) const; - int clear (::rocksdb::ColumnFamilyHandle * column_family); - - void open (bool & error_a, boost::filesystem::path const & path_a, bool open_read_only_a, ::rocksdb::Options const & options_a, std::vector<::rocksdb::ColumnFamilyDescriptor> column_families); - - bool do_upgrades (nano::write_transaction const &); - void upgrade_v21_to_v22 (nano::write_transaction const &); - - void construct_column_family_mutexes (); - ::rocksdb::Options get_db_options (); - ::rocksdb::ColumnFamilyOptions get_common_cf_options (std::shared_ptr<::rocksdb::TableFactory> const & table_factory_a, unsigned long long memtable_size_bytes_a) const; - ::rocksdb::ColumnFamilyOptions get_active_cf_options (std::shared_ptr<::rocksdb::TableFactory> const & table_factory_a, unsigned long long memtable_size_bytes_a) const; - ::rocksdb::ColumnFamilyOptions get_small_cf_options (std::shared_ptr<::rocksdb::TableFactory> const & table_factory_a) const; - ::rocksdb::BlockBasedTableOptions get_active_table_options (std::size_t lru_size) const; - ::rocksdb::BlockBasedTableOptions get_small_table_options () const; - ::rocksdb::ColumnFamilyOptions get_cf_options (std::string const & cf_name_a) const; - - void on_flush (::rocksdb::FlushJobInfo const &); - void flush_table (nano::tables table_a); - void flush_tombstones_check (nano::tables table_a); - void generate_tombstone_map (); - std::unordered_map create_cf_name_table_map () const; - - std::vector<::rocksdb::ColumnFamilyDescriptor> create_column_families (); - unsigned long long base_memtable_size_bytes () const; - unsigned long long blocks_memtable_size_bytes () const; - - constexpr static int base_memtable_size = 16; - constexpr static int base_block_cache_size = 8; - - friend class nano::rocksdb_block_store_tombstone_count_Test; - friend class nano::rocksdb::rocksdb_block_store_upgrade_v21_v22_Test; - }; -} // namespace rocksdb -} // namespace nano diff --git a/nano/node/rocksdb/rocksdb_txn.cpp b/nano/node/rocksdb/rocksdb_txn.cpp deleted file mode 100644 index 8bdeada2a0..0000000000 --- a/nano/node/rocksdb/rocksdb_txn.cpp +++ /dev/null @@ -1,109 +0,0 @@ -#include - -nano::read_rocksdb_txn::read_rocksdb_txn (rocksdb::DB * db_a) : - db (db_a) -{ - if (db_a) - { - options.snapshot = db_a->GetSnapshot (); - } -} - -nano::read_rocksdb_txn::~read_rocksdb_txn () -{ - reset (); -} - -void nano::read_rocksdb_txn::reset () -{ - if (db) - { - db->ReleaseSnapshot (options.snapshot); - } -} - -void nano::read_rocksdb_txn::renew () -{ - options.snapshot = db->GetSnapshot (); -} - -void * nano::read_rocksdb_txn::get_handle () const -{ - return (void *)&options; -} - -nano::write_rocksdb_txn::write_rocksdb_txn (rocksdb::OptimisticTransactionDB * db_a, std::vector const & tables_requiring_locks_a, std::vector const & tables_no_locks_a, std::unordered_map & mutexes_a) : - db (db_a), - tables_requiring_locks (tables_requiring_locks_a), - tables_no_locks (tables_no_locks_a), - mutexes (mutexes_a) -{ - lock (); - rocksdb::OptimisticTransactionOptions txn_options; - txn_options.set_snapshot = true; - txn = db->BeginTransaction (rocksdb::WriteOptions (), txn_options); -} - -nano::write_rocksdb_txn::~write_rocksdb_txn () -{ - commit (); - delete txn; - unlock (); -} - -void nano::write_rocksdb_txn::commit () -{ - if (active) - { - auto status = txn->Commit (); - - // If there are no available memtables try again a few more times - constexpr auto num_attempts = 10; - auto attempt_num = 0; - while (status.IsTryAgain () && attempt_num < num_attempts) - { - status = txn->Commit (); - ++attempt_num; - } - - if (!status.ok ()) - { - release_assert (false && "Unable to write to the RocksDB database", status.ToString ()); - } - active = false; - } -} - -void nano::write_rocksdb_txn::renew () -{ - rocksdb::OptimisticTransactionOptions txn_options; - txn_options.set_snapshot = true; - db->BeginTransaction (rocksdb::WriteOptions (), txn_options, txn); - active = true; -} - -void * nano::write_rocksdb_txn::get_handle () const -{ - return txn; -} - -void nano::write_rocksdb_txn::lock () -{ - for (auto table : tables_requiring_locks) - { - mutexes.at (table).lock (); - } -} - -void nano::write_rocksdb_txn::unlock () -{ - for (auto table : tables_requiring_locks) - { - mutexes.at (table).unlock (); - } -} - -bool nano::write_rocksdb_txn::contains (nano::tables table_a) const -{ - return (std::find (tables_requiring_locks.begin (), tables_requiring_locks.end (), table_a) != tables_requiring_locks.end ()) || (std::find (tables_no_locks.begin (), tables_no_locks.end (), table_a) != tables_no_locks.end ()); -} diff --git a/nano/node/rocksdb/version_store.hpp b/nano/node/rocksdb/version_store.hpp deleted file mode 100644 index acf252ca70..0000000000 --- a/nano/node/rocksdb/version_store.hpp +++ /dev/null @@ -1,21 +0,0 @@ -#pragma once - -#include - -namespace nano -{ -namespace rocksdb -{ - class store; - class version_store : public nano::version_store - { - protected: - nano::rocksdb::store & store; - - public: - explicit version_store (nano::rocksdb::store & store_a); - void put (nano::write_transaction const & transaction_a, int version_a) override; - int get (nano::transaction const & transaction_a) const override; - }; -} -} diff --git a/nano/node/scheduler/optimistic.cpp b/nano/node/scheduler/optimistic.cpp index a00fa4d96c..b47ddb9fdf 100644 --- a/nano/node/scheduler/optimistic.cpp +++ b/nano/node/scheduler/optimistic.cpp @@ -144,7 +144,7 @@ void nano::scheduler::optimistic::run () } } -void nano::scheduler::optimistic::run_one (nano::transaction const & transaction, entry const & candidate) +void nano::scheduler::optimistic::run_one (store::transaction const & transaction, entry const & candidate) { auto block = ledger.head_block (transaction, candidate.account); if (block) diff --git a/nano/node/scheduler/optimistic.hpp b/nano/node/scheduler/optimistic.hpp index 3c16c72059..8794789e79 100644 --- a/nano/node/scheduler/optimistic.hpp +++ b/nano/node/scheduler/optimistic.hpp @@ -70,7 +70,7 @@ class optimistic final bool predicate () const; void run (); - void run_one (nano::transaction const &, entry const & candidate); + void run_one (store::transaction const &, entry const & candidate); private: // Dependencies optimistic_config const & config; diff --git a/nano/node/scheduler/priority.cpp b/nano/node/scheduler/priority.cpp index e4bd01392e..c75c5b0976 100644 --- a/nano/node/scheduler/priority.cpp +++ b/nano/node/scheduler/priority.cpp @@ -35,7 +35,7 @@ void nano::scheduler::priority::stop () nano::join_or_pass (thread); } -bool nano::scheduler::priority::activate (nano::account const & account_a, nano::transaction const & transaction) +bool nano::scheduler::priority::activate (nano::account const & account_a, store::transaction const & transaction) { debug_assert (!account_a.is_zero ()); auto info = node.ledger.account_info (transaction, account_a); diff --git a/nano/node/scheduler/priority.hpp b/nano/node/scheduler/priority.hpp index 13b32b950b..30bf33ad31 100644 --- a/nano/node/scheduler/priority.hpp +++ b/nano/node/scheduler/priority.hpp @@ -36,7 +36,7 @@ class priority final * Activates the first unconfirmed block of \p account_a * @return true if account was activated */ - bool activate (nano::account const &, nano::transaction const &); + bool activate (nano::account const &, store::transaction const &); void notify (); std::size_t size () const; bool empty () const; diff --git a/nano/node/signatures.hpp b/nano/node/signatures.hpp index f1aa9f90cd..601948707e 100644 --- a/nano/node/signatures.hpp +++ b/nano/node/signatures.hpp @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include diff --git a/nano/node/unchecked_map.cpp b/nano/node/unchecked_map.cpp index 64b2b9cf0c..45d3b49563 100644 --- a/nano/node/unchecked_map.cpp +++ b/nano/node/unchecked_map.cpp @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include #include diff --git a/nano/node/vote_processor.hpp b/nano/node/vote_processor.hpp index 16c72b2e0b..0ffb39dd00 100644 --- a/nano/node/vote_processor.hpp +++ b/nano/node/vote_processor.hpp @@ -13,7 +13,10 @@ namespace nano { class signature_checker; class active_transactions; -class store; +namespace store +{ + class component; +} class node_observers; class stats; class node_config; @@ -25,7 +28,6 @@ class network_params; class node_flags; class stats; -class transaction; namespace transport { class channel; diff --git a/nano/node/voting.cpp b/nano/node/voting.cpp index 97e77bcb96..b8f1dba707 100644 --- a/nano/node/voting.cpp +++ b/nano/node/voting.cpp @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include @@ -184,7 +184,7 @@ nano::vote_generator::~vote_generator () stop (); } -void nano::vote_generator::process (nano::write_transaction const & transaction, nano::root const & root_a, nano::block_hash const & hash_a) +void nano::vote_generator::process (store::write_transaction const & transaction, nano::root const & root_a, nano::block_hash const & hash_a) { bool should_vote = false; if (is_final) diff --git a/nano/node/voting.hpp b/nano/node/voting.hpp index 7cde662d3c..13f113b106 100644 --- a/nano/node/voting.hpp +++ b/nano/node/voting.hpp @@ -144,7 +144,7 @@ class vote_generator final * Check if block is eligible for vote generation, then generates a vote or broadcasts votes already in cache * @param transaction : needs `tables::final_votes` lock */ - void process (nano::write_transaction const &, nano::root const &, nano::block_hash const &); + void process (store::write_transaction const &, nano::root const &, nano::block_hash const &); private: std::function const &, std::shared_ptr &)> reply_action; // must be set only during initialization by using set_reply_action diff --git a/nano/node/wallet.cpp b/nano/node/wallet.cpp index 804fe627e2..8f2ec7e0ca 100644 --- a/nano/node/wallet.cpp +++ b/nano/node/wallet.cpp @@ -2,9 +2,9 @@ #include #include #include -#include #include #include +#include #include #include @@ -15,19 +15,19 @@ #include -nano::uint256_union nano::wallet_store::check (nano::transaction const & transaction_a) +nano::uint256_union nano::wallet_store::check (store::transaction const & transaction_a) { nano::wallet_value value (entry_get_raw (transaction_a, nano::wallet_store::check_special)); return value.key; } -nano::uint256_union nano::wallet_store::salt (nano::transaction const & transaction_a) +nano::uint256_union nano::wallet_store::salt (store::transaction const & transaction_a) { nano::wallet_value value (entry_get_raw (transaction_a, nano::wallet_store::salt_special)); return value.key; } -void nano::wallet_store::wallet_key (nano::raw_key & prv_a, nano::transaction const & transaction_a) +void nano::wallet_store::wallet_key (nano::raw_key & prv_a, store::transaction const & transaction_a) { nano::lock_guard lock{ mutex }; nano::raw_key wallet_l; @@ -37,7 +37,7 @@ void nano::wallet_store::wallet_key (nano::raw_key & prv_a, nano::transaction co prv_a.decrypt (wallet_l, password_l, salt (transaction_a).owords[0]); } -void nano::wallet_store::seed (nano::raw_key & prv_a, nano::transaction const & transaction_a) +void nano::wallet_store::seed (nano::raw_key & prv_a, store::transaction const & transaction_a) { nano::wallet_value value (entry_get_raw (transaction_a, nano::wallet_store::seed_special)); nano::raw_key password_l; @@ -45,7 +45,7 @@ void nano::wallet_store::seed (nano::raw_key & prv_a, nano::transaction const & prv_a.decrypt (value.key, password_l, salt (transaction_a).owords[seed_iv_index]); } -void nano::wallet_store::seed_set (nano::transaction const & transaction_a, nano::raw_key const & prv_a) +void nano::wallet_store::seed_set (store::transaction const & transaction_a, nano::raw_key const & prv_a) { nano::raw_key password_l; wallet_key (password_l, transaction_a); @@ -55,7 +55,7 @@ void nano::wallet_store::seed_set (nano::transaction const & transaction_a, nano deterministic_clear (transaction_a); } -nano::public_key nano::wallet_store::deterministic_insert (nano::transaction const & transaction_a) +nano::public_key nano::wallet_store::deterministic_insert (store::transaction const & transaction_a) { auto index (deterministic_index_get (transaction_a)); auto prv = deterministic_key (transaction_a, index); @@ -75,7 +75,7 @@ nano::public_key nano::wallet_store::deterministic_insert (nano::transaction con return result; } -nano::public_key nano::wallet_store::deterministic_insert (nano::transaction const & transaction_a, uint32_t const index) +nano::public_key nano::wallet_store::deterministic_insert (store::transaction const & transaction_a, uint32_t const index) { auto prv = deterministic_key (transaction_a, index); nano::public_key result (nano::pub_key (prv)); @@ -86,7 +86,7 @@ nano::public_key nano::wallet_store::deterministic_insert (nano::transaction con return result; } -nano::raw_key nano::wallet_store::deterministic_key (nano::transaction const & transaction_a, uint32_t index_a) +nano::raw_key nano::wallet_store::deterministic_key (store::transaction const & transaction_a, uint32_t index_a) { debug_assert (valid_password (transaction_a)); nano::raw_key seed_l; @@ -94,20 +94,20 @@ nano::raw_key nano::wallet_store::deterministic_key (nano::transaction const & t return nano::deterministic_key (seed_l, index_a); } -uint32_t nano::wallet_store::deterministic_index_get (nano::transaction const & transaction_a) +uint32_t nano::wallet_store::deterministic_index_get (store::transaction const & transaction_a) { nano::wallet_value value (entry_get_raw (transaction_a, nano::wallet_store::deterministic_index_special)); return static_cast (value.key.number () & static_cast (-1)); } -void nano::wallet_store::deterministic_index_set (nano::transaction const & transaction_a, uint32_t index_a) +void nano::wallet_store::deterministic_index_set (store::transaction const & transaction_a, uint32_t index_a) { nano::raw_key index_l (index_a); nano::wallet_value value (index_l, 0); entry_put_raw (transaction_a, nano::wallet_store::deterministic_index_special, value); } -void nano::wallet_store::deterministic_clear (nano::transaction const & transaction_a) +void nano::wallet_store::deterministic_clear (store::transaction const & transaction_a) { nano::uint256_union key (0); for (auto i (begin (transaction_a)), n (end ()); i != n;) @@ -131,7 +131,7 @@ void nano::wallet_store::deterministic_clear (nano::transaction const & transact deterministic_index_set (transaction_a, 0); } -bool nano::wallet_store::valid_password (nano::transaction const & transaction_a) +bool nano::wallet_store::valid_password (store::transaction const & transaction_a) { nano::raw_key zero; zero.clear (); @@ -143,7 +143,7 @@ bool nano::wallet_store::valid_password (nano::transaction const & transaction_a return ok; } -bool nano::wallet_store::attempt_password (nano::transaction const & transaction_a, std::string const & password_a) +bool nano::wallet_store::attempt_password (store::transaction const & transaction_a, std::string const & password_a) { bool result = false; { @@ -166,7 +166,7 @@ bool nano::wallet_store::attempt_password (nano::transaction const & transaction return result; } -bool nano::wallet_store::rekey (nano::transaction const & transaction_a, std::string const & password_a) +bool nano::wallet_store::rekey (store::transaction const & transaction_a, std::string const & password_a) { nano::lock_guard lock{ mutex }; bool result (false); @@ -193,7 +193,7 @@ bool nano::wallet_store::rekey (nano::transaction const & transaction_a, std::st return result; } -void nano::wallet_store::derive_key (nano::raw_key & prv_a, nano::transaction const & transaction_a, std::string const & password_a) +void nano::wallet_store::derive_key (nano::raw_key & prv_a, store::transaction const & transaction_a, std::string const & password_a) { auto salt_l (salt (transaction_a)); kdf.phs (prv_a, password_a, salt_l); @@ -255,7 +255,7 @@ int const nano::wallet_store::special_count (7); std::size_t const nano::wallet_store::check_iv_index (0); std::size_t const nano::wallet_store::seed_iv_index (1); -nano::wallet_store::wallet_store (bool & init_a, nano::kdf & kdf_a, nano::transaction & transaction_a, nano::account representative_a, unsigned fanout_a, std::string const & wallet_a, std::string const & json_a) : +nano::wallet_store::wallet_store (bool & init_a, nano::kdf & kdf_a, store::transaction & transaction_a, nano::account representative_a, unsigned fanout_a, std::string const & wallet_a, std::string const & json_a) : password (0, fanout_a), wallet_key_mem (0, fanout_a), kdf (kdf_a) @@ -265,7 +265,7 @@ nano::wallet_store::wallet_store (bool & init_a, nano::kdf & kdf_a, nano::transa if (!init_a) { MDB_val junk; - debug_assert (mdb_get (tx (transaction_a), handle, nano::mdb_val (version_special), &junk) == MDB_NOTFOUND); + debug_assert (mdb_get (tx (transaction_a), handle, nano::store::lmdb::db_val (version_special), &junk) == MDB_NOTFOUND); boost::property_tree::ptree wallet_l; std::stringstream istream (json_a); try @@ -298,11 +298,11 @@ nano::wallet_store::wallet_store (bool & init_a, nano::kdf & kdf_a, nano::transa init_a = true; } } - init_a |= mdb_get (tx (transaction_a), handle, nano::mdb_val (version_special), &junk) != 0; - init_a |= mdb_get (tx (transaction_a), handle, nano::mdb_val (wallet_key_special), &junk) != 0; - init_a |= mdb_get (tx (transaction_a), handle, nano::mdb_val (salt_special), &junk) != 0; - init_a |= mdb_get (tx (transaction_a), handle, nano::mdb_val (check_special), &junk) != 0; - init_a |= mdb_get (tx (transaction_a), handle, nano::mdb_val (representative_special), &junk) != 0; + init_a |= mdb_get (tx (transaction_a), handle, nano::store::lmdb::db_val (version_special), &junk) != 0; + init_a |= mdb_get (tx (transaction_a), handle, nano::store::lmdb::db_val (wallet_key_special), &junk) != 0; + init_a |= mdb_get (tx (transaction_a), handle, nano::store::lmdb::db_val (salt_special), &junk) != 0; + init_a |= mdb_get (tx (transaction_a), handle, nano::store::lmdb::db_val (check_special), &junk) != 0; + init_a |= mdb_get (tx (transaction_a), handle, nano::store::lmdb::db_val (representative_special), &junk) != 0; nano::raw_key key; key.clear (); password.value_set (key); @@ -311,7 +311,7 @@ nano::wallet_store::wallet_store (bool & init_a, nano::kdf & kdf_a, nano::transa } } -nano::wallet_store::wallet_store (bool & init_a, nano::kdf & kdf_a, nano::transaction & transaction_a, nano::account representative_a, unsigned fanout_a, std::string const & wallet_a) : +nano::wallet_store::wallet_store (bool & init_a, nano::kdf & kdf_a, store::transaction & transaction_a, nano::account representative_a, unsigned fanout_a, std::string const & wallet_a) : password (0, fanout_a), wallet_key_mem (0, fanout_a), kdf (kdf_a) @@ -322,7 +322,7 @@ nano::wallet_store::wallet_store (bool & init_a, nano::kdf & kdf_a, nano::transa { int version_status; MDB_val version_value; - version_status = mdb_get (tx (transaction_a), handle, nano::mdb_val (version_special), &version_value); + version_status = mdb_get (tx (transaction_a), handle, nano::store::lmdb::db_val (version_special), &version_value); if (version_status == MDB_NOTFOUND) { version_put (transaction_a, version_current); @@ -361,7 +361,7 @@ nano::wallet_store::wallet_store (bool & init_a, nano::kdf & kdf_a, nano::transa wallet_key_mem.value_set (key); } -std::vector nano::wallet_store::accounts (nano::transaction const & transaction_a) +std::vector nano::wallet_store::accounts (store::transaction const & transaction_a) { std::vector result; for (auto i (begin (transaction_a)), n (end ()); i != n; ++i) @@ -372,7 +372,7 @@ std::vector nano::wallet_store::accounts (nano::transaction const return result; } -void nano::wallet_store::initialize (nano::transaction const & transaction_a, bool & init_a, std::string const & path_a) +void nano::wallet_store::initialize (store::transaction const & transaction_a, bool & init_a, std::string const & path_a) { debug_assert (strlen (path_a.c_str ()) == path_a.size ()); auto error (0); @@ -382,25 +382,25 @@ void nano::wallet_store::initialize (nano::transaction const & transaction_a, bo init_a = error != 0; } -bool nano::wallet_store::is_representative (nano::transaction const & transaction_a) +bool nano::wallet_store::is_representative (store::transaction const & transaction_a) { return exists (transaction_a, representative (transaction_a)); } -void nano::wallet_store::representative_set (nano::transaction const & transaction_a, nano::account const & representative_a) +void nano::wallet_store::representative_set (store::transaction const & transaction_a, nano::account const & representative_a) { nano::raw_key rep; rep.bytes = representative_a.bytes; entry_put_raw (transaction_a, nano::wallet_store::representative_special, nano::wallet_value (rep, 0)); } -nano::account nano::wallet_store::representative (nano::transaction const & transaction_a) +nano::account nano::wallet_store::representative (store::transaction const & transaction_a) { nano::wallet_value value (entry_get_raw (transaction_a, nano::wallet_store::representative_special)); return reinterpret_cast (value.key); } -nano::public_key nano::wallet_store::insert_adhoc (nano::transaction const & transaction_a, nano::raw_key const & prv) +nano::public_key nano::wallet_store::insert_adhoc (store::transaction const & transaction_a, nano::raw_key const & prv) { debug_assert (valid_password (transaction_a)); nano::public_key pub (nano::pub_key (prv)); @@ -412,7 +412,7 @@ nano::public_key nano::wallet_store::insert_adhoc (nano::transaction const & tra return pub; } -bool nano::wallet_store::insert_watch (nano::transaction const & transaction_a, nano::account const & pub_a) +bool nano::wallet_store::insert_watch (store::transaction const & transaction_a, nano::account const & pub_a) { bool error (!valid_public_key (pub_a)); if (!error) @@ -422,18 +422,18 @@ bool nano::wallet_store::insert_watch (nano::transaction const & transaction_a, return error; } -void nano::wallet_store::erase (nano::transaction const & transaction_a, nano::account const & pub) +void nano::wallet_store::erase (store::transaction const & transaction_a, nano::account const & pub) { - auto status (mdb_del (tx (transaction_a), handle, nano::mdb_val (pub), nullptr)); + auto status (mdb_del (tx (transaction_a), handle, nano::store::lmdb::db_val (pub), nullptr)); (void)status; debug_assert (status == 0); } -nano::wallet_value nano::wallet_store::entry_get_raw (nano::transaction const & transaction_a, nano::account const & pub_a) +nano::wallet_value nano::wallet_store::entry_get_raw (store::transaction const & transaction_a, nano::account const & pub_a) { nano::wallet_value result; - nano::mdb_val value; - auto status (mdb_get (tx (transaction_a), handle, nano::mdb_val (pub_a), value)); + nano::store::lmdb::db_val value; + auto status (mdb_get (tx (transaction_a), handle, nano::store::lmdb::db_val (pub_a), value)); if (status == 0) { result = nano::wallet_value (value); @@ -446,9 +446,9 @@ nano::wallet_value nano::wallet_store::entry_get_raw (nano::transaction const & return result; } -void nano::wallet_store::entry_put_raw (nano::transaction const & transaction_a, nano::account const & pub_a, nano::wallet_value const & entry_a) +void nano::wallet_store::entry_put_raw (store::transaction const & transaction_a, nano::account const & pub_a, nano::wallet_value const & entry_a) { - auto status (mdb_put (tx (transaction_a), handle, nano::mdb_val (pub_a), nano::mdb_val (sizeof (entry_a), const_cast (&entry_a)), 0)); + auto status (mdb_put (tx (transaction_a), handle, nano::store::lmdb::db_val (pub_a), nano::store::lmdb::db_val (sizeof (entry_a), const_cast (&entry_a)), 0)); (void)status; debug_assert (status == 0); } @@ -476,7 +476,7 @@ nano::key_type nano::wallet_store::key_type (nano::wallet_value const & value_a) return result; } -bool nano::wallet_store::fetch (nano::transaction const & transaction_a, nano::account const & pub, nano::raw_key & prv) +bool nano::wallet_store::fetch (store::transaction const & transaction_a, nano::account const & pub, nano::raw_key & prv) { auto result (false); if (valid_password (transaction_a)) @@ -534,15 +534,15 @@ bool nano::wallet_store::valid_public_key (nano::public_key const & pub) return pub.number () >= special_count; } -bool nano::wallet_store::exists (nano::transaction const & transaction_a, nano::public_key const & pub) +bool nano::wallet_store::exists (store::transaction const & transaction_a, nano::public_key const & pub) { return valid_public_key (pub) && find (transaction_a, pub) != end (); } -void nano::wallet_store::serialize_json (nano::transaction const & transaction_a, std::string & string_a) +void nano::wallet_store::serialize_json (store::transaction const & transaction_a, std::string & string_a) { boost::property_tree::ptree tree; - for (nano::store_iterator i (std::make_unique> (transaction_a, handle)), n (nullptr); i != n; ++i) + for (store::iterator i (std::make_unique> (transaction_a, handle)), n (nullptr); i != n; ++i) { tree.put (i->first.to_string (), i->second.key.to_string ()); } @@ -551,7 +551,7 @@ void nano::wallet_store::serialize_json (nano::transaction const & transaction_a string_a = ostream.str (); } -void nano::wallet_store::write_backup (nano::transaction const & transaction_a, boost::filesystem::path const & path_a) +void nano::wallet_store::write_backup (store::transaction const & transaction_a, boost::filesystem::path const & path_a) { std::ofstream backup_file; backup_file.open (path_a.string ()); @@ -567,7 +567,7 @@ void nano::wallet_store::write_backup (nano::transaction const & transaction_a, } } -bool nano::wallet_store::move (nano::transaction const & transaction_a, nano::wallet_store & other_a, std::vector const & keys) +bool nano::wallet_store::move (store::transaction const & transaction_a, nano::wallet_store & other_a, std::vector const & keys) { debug_assert (valid_password (transaction_a)); debug_assert (other_a.valid_password (transaction_a)); @@ -586,7 +586,7 @@ bool nano::wallet_store::move (nano::transaction const & transaction_a, nano::wa return result; } -bool nano::wallet_store::import (nano::transaction const & transaction_a, nano::wallet_store & other_a) +bool nano::wallet_store::import (store::transaction const & transaction_a, nano::wallet_store & other_a) { debug_assert (valid_password (transaction_a)); debug_assert (other_a.valid_password (transaction_a)); @@ -612,7 +612,7 @@ bool nano::wallet_store::import (nano::transaction const & transaction_a, nano:: return result; } -bool nano::wallet_store::work_get (nano::transaction const & transaction_a, nano::public_key const & pub_a, uint64_t & work_a) +bool nano::wallet_store::work_get (store::transaction const & transaction_a, nano::public_key const & pub_a, uint64_t & work_a) { auto result (false); auto entry (entry_get_raw (transaction_a, pub_a)); @@ -627,7 +627,7 @@ bool nano::wallet_store::work_get (nano::transaction const & transaction_a, nano return result; } -void nano::wallet_store::work_put (nano::transaction const & transaction_a, nano::public_key const & pub_a, uint64_t work_a) +void nano::wallet_store::work_put (store::transaction const & transaction_a, nano::public_key const & pub_a, uint64_t work_a) { auto entry (entry_get_raw (transaction_a, pub_a)); debug_assert (!entry.key.is_zero ()); @@ -635,7 +635,7 @@ void nano::wallet_store::work_put (nano::transaction const & transaction_a, nano entry_put_raw (transaction_a, pub_a, entry); } -unsigned nano::wallet_store::version (nano::transaction const & transaction_a) +unsigned nano::wallet_store::version (store::transaction const & transaction_a) { nano::wallet_value value (entry_get_raw (transaction_a, nano::wallet_store::version_special)); auto entry (value.key); @@ -643,7 +643,7 @@ unsigned nano::wallet_store::version (nano::transaction const & transaction_a) return result; } -void nano::wallet_store::version_put (nano::transaction const & transaction_a, unsigned version_a) +void nano::wallet_store::version_put (store::transaction const & transaction_a, unsigned version_a) { nano::raw_key entry (version_a); entry_put_raw (transaction_a, nano::wallet_store::version_special, nano::wallet_value (entry, 0)); @@ -657,14 +657,14 @@ void nano::kdf::phs (nano::raw_key & result_a, std::string const & password_a, n (void)success; } -nano::wallet::wallet (bool & init_a, nano::transaction & transaction_a, nano::wallets & wallets_a, std::string const & wallet_a) : +nano::wallet::wallet (bool & init_a, store::transaction & transaction_a, nano::wallets & wallets_a, std::string const & wallet_a) : lock_observer ([] (bool, bool) {}), store (init_a, wallets_a.kdf, transaction_a, wallets_a.node.config.random_representative (), wallets_a.node.config.password_fanout, wallet_a), wallets (wallets_a) { } -nano::wallet::wallet (bool & init_a, nano::transaction & transaction_a, nano::wallets & wallets_a, std::string const & wallet_a, std::string const & json) : +nano::wallet::wallet (bool & init_a, store::transaction & transaction_a, nano::wallets & wallets_a, std::string const & wallet_a, std::string const & json) : lock_observer ([] (bool, bool) {}), store (init_a, wallets_a.kdf, transaction_a, wallets_a.node.config.random_representative (), wallets_a.node.config.password_fanout, wallet_a, json), wallets (wallets_a) @@ -693,7 +693,7 @@ void nano::wallet::enter_initial_password () } } -bool nano::wallet::enter_password (nano::transaction const & transaction_a, std::string const & password_a) +bool nano::wallet::enter_password (store::transaction const & transaction_a, std::string const & password_a) { auto result (store.attempt_password (transaction_a, password_a)); if (!result) @@ -712,7 +712,7 @@ bool nano::wallet::enter_password (nano::transaction const & transaction_a, std: return result; } -nano::public_key nano::wallet::deterministic_insert (nano::transaction const & transaction_a, bool generate_work_a) +nano::public_key nano::wallet::deterministic_insert (store::transaction const & transaction_a, bool generate_work_a) { nano::public_key key{}; if (store.valid_password (transaction_a)) @@ -779,7 +779,7 @@ nano::public_key nano::wallet::insert_adhoc (nano::raw_key const & key_a, bool g return key; } -bool nano::wallet::insert_watch (nano::transaction const & transaction_a, nano::public_key const & pub_a) +bool nano::wallet::insert_watch (store::transaction const & transaction_a, nano::public_key const & pub_a) { return store.insert_watch (transaction_a, pub_a); } @@ -820,7 +820,7 @@ void nano::wallet::serialize (std::string & json_a) store.serialize_json (transaction, json_a); } -void nano::wallet_store::destroy (nano::transaction const & transaction_a) +void nano::wallet_store::destroy (store::transaction const & transaction_a) { auto status (mdb_drop (tx (transaction_a), handle, 1)); (void)status; @@ -932,10 +932,10 @@ std::shared_ptr nano::wallet::change_action (nano::account const & std::shared_ptr nano::wallet::send_action (nano::account const & source_a, nano::account const & account_a, nano::uint128_t const & amount_a, uint64_t work_a, bool generate_work_a, boost::optional id_a) { - boost::optional id_mdb_val; + boost::optional id_mdb_val; if (id_a) { - id_mdb_val = nano::mdb_val (id_a->size (), const_cast (id_a->data ())); + id_mdb_val = nano::store::lmdb::db_val (id_a->size (), const_cast (id_a->data ())); } auto prepare_send = [&id_mdb_val, &wallets = this->wallets, &store = this->store, &source_a, &amount_a, &work_a, &account_a] (auto const & transaction) { @@ -947,7 +947,7 @@ std::shared_ptr nano::wallet::send_action (nano::account const & so details.is_send = true; if (id_mdb_val) { - nano::mdb_val result; + nano::store::lmdb::db_val result; auto status (mdb_get (wallets.env.tx (transaction), wallets.node.wallets.send_action_ids, *id_mdb_val, result)); if (status == 0) { @@ -988,7 +988,7 @@ std::shared_ptr nano::wallet::send_action (nano::account const & so details.epoch = info->epoch (); if (id_mdb_val && block != nullptr) { - auto status (mdb_put (wallets.env.tx (transaction), wallets.node.wallets.send_action_ids, *id_mdb_val, nano::mdb_val (block->hash ()), 0)); + auto status (mdb_put (wallets.env.tx (transaction), wallets.node.wallets.send_action_ids, *id_mdb_val, nano::store::lmdb::db_val (block->hash ()), 0)); if (status != 0) { block = nullptr; @@ -1125,7 +1125,7 @@ void nano::wallet::send_async (nano::account const & source_a, nano::account con } // Update work for account if latest root is root_a -void nano::wallet::work_update (nano::transaction const & transaction_a, nano::account const & account_a, nano::root const & root_a, uint64_t work_a) +void nano::wallet::work_update (store::transaction const & transaction_a, nano::account const & account_a, nano::root const & root_a, uint64_t work_a) { debug_assert (!wallets.node.network_params.work.validate_entry (nano::work_version::work_1, root_a, work_a)); debug_assert (store.exists (transaction_a, account_a)); @@ -1161,7 +1161,7 @@ void nano::wallet::work_ensure (nano::account const & account_a, nano::root cons }); } -bool nano::wallet::search_receivable (nano::transaction const & wallet_transaction_a) +bool nano::wallet::search_receivable (store::transaction const & wallet_transaction_a) { auto result (!store.valid_password (wallet_transaction_a)); if (!result) @@ -1211,7 +1211,7 @@ bool nano::wallet::search_receivable (nano::transaction const & wallet_transacti return result; } -void nano::wallet::init_free_accounts (nano::transaction const & transaction_a) +void nano::wallet::init_free_accounts (store::transaction const & transaction_a) { free_accounts.clear (); for (auto i (store.begin (transaction_a)), n (store.end ()); i != n; ++i) @@ -1220,7 +1220,7 @@ void nano::wallet::init_free_accounts (nano::transaction const & transaction_a) } } -uint32_t nano::wallet::deterministic_check (nano::transaction const & transaction_a, uint32_t index) +uint32_t nano::wallet::deterministic_check (store::transaction const & transaction_a, uint32_t index) { auto block_transaction (wallets.node.store.tx_begin_read ()); for (uint32_t i (index + 1), n (index + 64); i < n; ++i) @@ -1250,7 +1250,7 @@ uint32_t nano::wallet::deterministic_check (nano::transaction const & transactio return index; } -nano::public_key nano::wallet::change_seed (nano::transaction const & transaction_a, nano::raw_key const & prv_a, uint32_t count) +nano::public_key nano::wallet::change_seed (store::transaction const & transaction_a, nano::raw_key const & prv_a, uint32_t count) { store.seed_set (transaction_a, prv_a); auto account = deterministic_insert (transaction_a); @@ -1266,7 +1266,7 @@ nano::public_key nano::wallet::change_seed (nano::transaction const & transactio return account; } -void nano::wallet::deterministic_restore (nano::transaction const & transaction_a) +void nano::wallet::deterministic_restore (store::transaction const & transaction_a) { auto index (store.deterministic_index_get (transaction_a)); auto new_index (deterministic_check (transaction_a, index)); @@ -1348,8 +1348,8 @@ nano::wallets::wallets (bool error_a, nano::node & node_a) : release_assert (status == 0); std::string beginning (nano::uint256_union (0).to_string ()); std::string end ((nano::uint256_union (nano::uint256_t (0) - nano::uint256_t (1))).to_string ()); - nano::store_iterator, nano::no_value> i (std::make_unique, nano::no_value>> (transaction, handle, nano::mdb_val (beginning.size (), const_cast (beginning.c_str ())))); - nano::store_iterator, nano::no_value> n (std::make_unique, nano::no_value>> (transaction, handle, nano::mdb_val (end.size (), const_cast (end.c_str ())))); + store::iterator, nano::no_value> i (std::make_unique, nano::no_value>> (transaction, handle, nano::store::lmdb::db_val (beginning.size (), const_cast (beginning.c_str ())))); + store::iterator, nano::no_value> n (std::make_unique, nano::no_value>> (transaction, handle, nano::store::lmdb::db_val (end.size (), const_cast (end.c_str ())))); for (; i != n; ++i) { nano::wallet_id id; @@ -1387,7 +1387,7 @@ nano::wallets::wallets (bool error_a, nano::node & node_a) : char const * store_path; mdb_env_get_path (env, &store_path); boost::filesystem::path const path (store_path); - nano::lmdb::store::create_backup_file (env, path, node_a.logger); + nano::store::lmdb::component::create_backup_file (env, path, node_a.logger); } for (auto & item : items) { @@ -1477,8 +1477,8 @@ void nano::wallets::reload () std::unordered_set stored_items; std::string beginning (nano::uint256_union (0).to_string ()); std::string end ((nano::uint256_union (nano::uint256_t (0) - nano::uint256_t (1))).to_string ()); - nano::store_iterator, nano::no_value> i (std::make_unique, nano::no_value>> (transaction, handle, nano::mdb_val (beginning.size (), const_cast (beginning.c_str ())))); - nano::store_iterator, nano::no_value> n (std::make_unique, nano::no_value>> (transaction, handle, nano::mdb_val (end.size (), const_cast (end.c_str ())))); + store::iterator, nano::no_value> i (std::make_unique, nano::no_value>> (transaction, handle, nano::store::lmdb::db_val (beginning.size (), const_cast (beginning.c_str ())))); + store::iterator, nano::no_value> n (std::make_unique, nano::no_value>> (transaction, handle, nano::store::lmdb::db_val (end.size (), const_cast (end.c_str ())))); for (; i != n; ++i) { nano::wallet_id id; @@ -1574,7 +1574,7 @@ void nano::wallets::foreach_representative (std::function lock{ mutex }; auto result (false); @@ -1607,17 +1607,17 @@ void nano::wallets::start () } }; } -nano::write_transaction nano::wallets::tx_begin_write () +nano::store::write_transaction nano::wallets::tx_begin_write () { return env.tx_begin_write (); } -nano::read_transaction nano::wallets::tx_begin_read () +nano::store::read_transaction nano::wallets::tx_begin_read () { return env.tx_begin_read (); } -void nano::wallets::clear_send_ids (nano::transaction const & transaction_a) +void nano::wallets::clear_send_ids (store::transaction const & transaction_a) { auto status (mdb_drop (env.tx (transaction_a), send_action_ids, 0)); (void)status; @@ -1696,9 +1696,9 @@ void nano::wallets::ongoing_compute_reps () }); } -void nano::wallets::split_if_needed (nano::transaction & transaction_destination, nano::store & store_a) +void nano::wallets::split_if_needed (store::transaction & transaction_destination, nano::store::component & store_a) { - auto store_l = dynamic_cast (&store_a); + auto store_l = dynamic_cast (&store_a); if (store_l != nullptr) { if (items.empty ()) @@ -1706,8 +1706,8 @@ void nano::wallets::split_if_needed (nano::transaction & transaction_destination std::string beginning (nano::uint256_union (0).to_string ()); std::string end ((nano::uint256_union (nano::uint256_t (0) - nano::uint256_t (1))).to_string ()); - auto get_store_it = [&handle = handle] (nano::transaction const & transaction_source, std::string const & hash) { - return nano::store_iterator, nano::no_value> (std::make_unique, nano::no_value>> (transaction_source, handle, nano::mdb_val (hash.size (), const_cast (hash.c_str ())))); + auto get_store_it = [&handle = handle] (store::transaction const & transaction_source, std::string const & hash) { + return store::iterator, nano::no_value> (std::make_unique, nano::no_value>> (transaction_source, handle, nano::store::lmdb::db_val (hash.size (), const_cast (hash.c_str ())))); }; // First do a read pass to check if there are any wallets that need extracting (to save holding a write lock and potentially being blocked) @@ -1779,22 +1779,22 @@ std::unordered_map> nano::wallets nano::uint128_t const nano::wallets::generate_priority = std::numeric_limits::max (); nano::uint128_t const nano::wallets::high_priority = std::numeric_limits::max () - 1; -nano::store_iterator nano::wallet_store::begin (nano::transaction const & transaction_a) +nano::store::iterator nano::wallet_store::begin (store::transaction const & transaction_a) { - nano::store_iterator result (std::make_unique> (transaction_a, handle, nano::mdb_val (nano::account (special_count)))); + store::iterator result (std::make_unique> (transaction_a, handle, nano::store::lmdb::db_val (nano::account (special_count)))); return result; } -nano::store_iterator nano::wallet_store::begin (nano::transaction const & transaction_a, nano::account const & key) +nano::store::iterator nano::wallet_store::begin (store::transaction const & transaction_a, nano::account const & key) { - nano::store_iterator result (std::make_unique> (transaction_a, handle, nano::mdb_val (key))); + store::iterator result (std::make_unique> (transaction_a, handle, nano::store::lmdb::db_val (key))); return result; } -nano::store_iterator nano::wallet_store::find (nano::transaction const & transaction_a, nano::account const & key) +nano::store::iterator nano::wallet_store::find (store::transaction const & transaction_a, nano::account const & key) { auto result (begin (transaction_a, key)); - nano::store_iterator end (nullptr); + store::iterator end (nullptr); if (result != end) { if (result->first == key) @@ -1813,12 +1813,12 @@ nano::store_iterator nano::wallet_store::find return result; } -nano::store_iterator nano::wallet_store::end () +nano::store::iterator nano::wallet_store::end () { - return nano::store_iterator (nullptr); + return store::iterator (nullptr); } nano::mdb_wallets_store::mdb_wallets_store (boost::filesystem::path const & path_a, nano::lmdb_config const & lmdb_config_a) : - environment (error, path_a, nano::mdb_env::options::make ().set_config (lmdb_config_a).override_config_sync (nano::lmdb_config::sync_strategy::always).override_config_map_size (1ULL * 1024 * 1024 * 1024)) + environment (error, path_a, nano::store::lmdb::env::options::make ().set_config (lmdb_config_a).override_config_sync (nano::lmdb_config::sync_strategy::always).override_config_map_size (1ULL * 1024 * 1024 * 1024)) { } @@ -1827,7 +1827,7 @@ bool nano::mdb_wallets_store::init_error () const return error; } -MDB_txn * nano::wallet_store::tx (nano::transaction const & transaction_a) const +MDB_txn * nano::wallet_store::tx (store::transaction const & transaction_a) const { return static_cast (transaction_a.get_handle ()); } diff --git a/nano/node/wallet.hpp b/nano/node/wallet.hpp index 32016486dc..c585ae2995 100644 --- a/nano/node/wallet.hpp +++ b/nano/node/wallet.hpp @@ -3,11 +3,11 @@ #include #include #include -#include -#include #include #include -#include +#include +#include +#include #include #include @@ -52,50 +52,50 @@ enum class key_type class wallet_store final { public: - wallet_store (bool &, nano::kdf &, nano::transaction &, nano::account, unsigned, std::string const &); - wallet_store (bool &, nano::kdf &, nano::transaction &, nano::account, unsigned, std::string const &, std::string const &); - std::vector accounts (nano::transaction const &); - void initialize (nano::transaction const &, bool &, std::string const &); - nano::uint256_union check (nano::transaction const &); - bool rekey (nano::transaction const &, std::string const &); - bool valid_password (nano::transaction const &); + wallet_store (bool &, nano::kdf &, store::transaction &, nano::account, unsigned, std::string const &); + wallet_store (bool &, nano::kdf &, store::transaction &, nano::account, unsigned, std::string const &, std::string const &); + std::vector accounts (store::transaction const &); + void initialize (store::transaction const &, bool &, std::string const &); + nano::uint256_union check (store::transaction const &); + bool rekey (store::transaction const &, std::string const &); + bool valid_password (store::transaction const &); bool valid_public_key (nano::public_key const &); - bool attempt_password (nano::transaction const &, std::string const &); - void wallet_key (nano::raw_key &, nano::transaction const &); - void seed (nano::raw_key &, nano::transaction const &); - void seed_set (nano::transaction const &, nano::raw_key const &); + bool attempt_password (store::transaction const &, std::string const &); + void wallet_key (nano::raw_key &, store::transaction const &); + void seed (nano::raw_key &, store::transaction const &); + void seed_set (store::transaction const &, nano::raw_key const &); nano::key_type key_type (nano::wallet_value const &); - nano::public_key deterministic_insert (nano::transaction const &); - nano::public_key deterministic_insert (nano::transaction const &, uint32_t const); - nano::raw_key deterministic_key (nano::transaction const &, uint32_t); - uint32_t deterministic_index_get (nano::transaction const &); - void deterministic_index_set (nano::transaction const &, uint32_t); - void deterministic_clear (nano::transaction const &); - nano::uint256_union salt (nano::transaction const &); - bool is_representative (nano::transaction const &); - nano::account representative (nano::transaction const &); - void representative_set (nano::transaction const &, nano::account const &); - nano::public_key insert_adhoc (nano::transaction const &, nano::raw_key const &); - bool insert_watch (nano::transaction const &, nano::account const &); - void erase (nano::transaction const &, nano::account const &); - nano::wallet_value entry_get_raw (nano::transaction const &, nano::account const &); - void entry_put_raw (nano::transaction const &, nano::account const &, nano::wallet_value const &); - bool fetch (nano::transaction const &, nano::account const &, nano::raw_key &); - bool exists (nano::transaction const &, nano::account const &); - void destroy (nano::transaction const &); - nano::store_iterator find (nano::transaction const &, nano::account const &); - nano::store_iterator begin (nano::transaction const &, nano::account const &); - nano::store_iterator begin (nano::transaction const &); - nano::store_iterator end (); - void derive_key (nano::raw_key &, nano::transaction const &, std::string const &); - void serialize_json (nano::transaction const &, std::string &); - void write_backup (nano::transaction const &, boost::filesystem::path const &); - bool move (nano::transaction const &, nano::wallet_store &, std::vector const &); - bool import (nano::transaction const &, nano::wallet_store &); - bool work_get (nano::transaction const &, nano::public_key const &, uint64_t &); - void work_put (nano::transaction const &, nano::public_key const &, uint64_t); - unsigned version (nano::transaction const &); - void version_put (nano::transaction const &, unsigned); + nano::public_key deterministic_insert (store::transaction const &); + nano::public_key deterministic_insert (store::transaction const &, uint32_t const); + nano::raw_key deterministic_key (store::transaction const &, uint32_t); + uint32_t deterministic_index_get (store::transaction const &); + void deterministic_index_set (store::transaction const &, uint32_t); + void deterministic_clear (store::transaction const &); + nano::uint256_union salt (store::transaction const &); + bool is_representative (store::transaction const &); + nano::account representative (store::transaction const &); + void representative_set (store::transaction const &, nano::account const &); + nano::public_key insert_adhoc (store::transaction const &, nano::raw_key const &); + bool insert_watch (store::transaction const &, nano::account const &); + void erase (store::transaction const &, nano::account const &); + nano::wallet_value entry_get_raw (store::transaction const &, nano::account const &); + void entry_put_raw (store::transaction const &, nano::account const &, nano::wallet_value const &); + bool fetch (store::transaction const &, nano::account const &, nano::raw_key &); + bool exists (store::transaction const &, nano::account const &); + void destroy (store::transaction const &); + store::iterator find (store::transaction const &, nano::account const &); + store::iterator begin (store::transaction const &, nano::account const &); + store::iterator begin (store::transaction const &); + store::iterator end (); + void derive_key (nano::raw_key &, store::transaction const &, std::string const &); + void serialize_json (store::transaction const &, std::string &); + void write_backup (store::transaction const &, boost::filesystem::path const &); + bool move (store::transaction const &, nano::wallet_store &, std::vector const &); + bool import (store::transaction const &, nano::wallet_store &); + bool work_get (store::transaction const &, nano::public_key const &, uint64_t &); + void work_put (store::transaction const &, nano::public_key const &, uint64_t); + unsigned version (store::transaction const &); + void version_put (store::transaction const &, unsigned); nano::fan password; nano::fan wallet_key_mem; static unsigned const version_1 = 1; @@ -118,7 +118,7 @@ class wallet_store final std::recursive_mutex mutex; private: - MDB_txn * tx (nano::transaction const &) const; + MDB_txn * tx (store::transaction const &) const; }; // A wallet is a set of account keys encrypted by a common encryption key class wallet final : public std::enable_shared_from_this @@ -128,13 +128,13 @@ class wallet final : public std::enable_shared_from_this std::shared_ptr receive_action (nano::block_hash const &, nano::account const &, nano::uint128_union const &, nano::account const &, uint64_t = 0, bool = true); std::shared_ptr send_action (nano::account const &, nano::account const &, nano::uint128_t const &, uint64_t = 0, bool = true, boost::optional = {}); bool action_complete (std::shared_ptr const &, nano::account const &, bool const, nano::block_details const &); - wallet (bool &, nano::transaction &, nano::wallets &, std::string const &); - wallet (bool &, nano::transaction &, nano::wallets &, std::string const &, std::string const &); + wallet (bool &, store::transaction &, nano::wallets &, std::string const &); + wallet (bool &, store::transaction &, nano::wallets &, std::string const &, std::string const &); void enter_initial_password (); - bool enter_password (nano::transaction const &, std::string const &); + bool enter_password (store::transaction const &, std::string const &); nano::public_key insert_adhoc (nano::raw_key const &, bool = true); - bool insert_watch (nano::transaction const &, nano::public_key const &); - nano::public_key deterministic_insert (nano::transaction const &, bool = true); + bool insert_watch (store::transaction const &, nano::public_key const &); + nano::public_key deterministic_insert (store::transaction const &, bool = true); nano::public_key deterministic_insert (uint32_t, bool = true); nano::public_key deterministic_insert (bool = true); bool exists (nano::public_key const &); @@ -147,15 +147,15 @@ class wallet final : public std::enable_shared_from_this nano::block_hash send_sync (nano::account const &, nano::account const &, nano::uint128_t const &); void send_async (nano::account const &, nano::account const &, nano::uint128_t const &, std::function const &)> const &, uint64_t = 0, bool = true, boost::optional = {}); void work_cache_blocking (nano::account const &, nano::root const &); - void work_update (nano::transaction const &, nano::account const &, nano::root const &, uint64_t); + void work_update (store::transaction const &, nano::account const &, nano::root const &, uint64_t); // Schedule work generation after a few seconds void work_ensure (nano::account const &, nano::root const &); - bool search_receivable (nano::transaction const &); - void init_free_accounts (nano::transaction const &); - uint32_t deterministic_check (nano::transaction const & transaction_a, uint32_t index); + bool search_receivable (store::transaction const &); + void init_free_accounts (store::transaction const &); + uint32_t deterministic_check (store::transaction const & transaction_a, uint32_t index); /** Changes the wallet seed and returns the first account */ - nano::public_key change_seed (nano::transaction const & transaction_a, nano::raw_key const & prv_a, uint32_t count = 0); - void deterministic_restore (nano::transaction const & transaction_a); + nano::public_key change_seed (store::transaction const & transaction_a, nano::raw_key const & prv_a, uint32_t count = 0); + void deterministic_restore (store::transaction const & transaction_a); bool live (); std::unordered_set free_accounts; std::function lock_observer; @@ -205,15 +205,15 @@ class wallets final void do_wallet_actions (); void queue_wallet_action (nano::uint128_t const &, std::shared_ptr const &, std::function); void foreach_representative (std::function const &); - bool exists (nano::transaction const &, nano::account const &); + bool exists (store::transaction const &, nano::account const &); void start (); void stop (); - void clear_send_ids (nano::transaction const &); + void clear_send_ids (store::transaction const &); nano::wallet_representatives reps () const; bool check_rep (nano::account const &, nano::uint128_t const &, bool const = true); void compute_reps (); void ongoing_compute_reps (); - void split_if_needed (nano::transaction &, nano::store &); + void split_if_needed (store::transaction &, nano::store::component &); void move_table (std::string const &, MDB_txn *, MDB_txn *); std::unordered_map> get_wallets (); nano::network_params & network_params; @@ -228,16 +228,16 @@ class wallets final MDB_dbi handle; MDB_dbi send_action_ids; nano::node & node; - nano::mdb_env & env; + nano::store::lmdb::env & env; std::atomic stopped; std::thread thread; static nano::uint128_t const generate_priority; static nano::uint128_t const high_priority; /** Start read-write transaction */ - nano::write_transaction tx_begin_write (); + store::write_transaction tx_begin_write (); /** Start read-only transaction */ - nano::read_transaction tx_begin_read (); + store::read_transaction tx_begin_read (); private: mutable nano::mutex reps_cache_mutex; @@ -256,7 +256,7 @@ class mdb_wallets_store final : public wallets_store { public: mdb_wallets_store (boost::filesystem::path const &, nano::lmdb_config const & lmdb_config_a = nano::lmdb_config{}); - nano::mdb_env environment; + nano::store::lmdb::env environment; bool init_error () const override; bool error{ false }; }; diff --git a/nano/qt/qt.cpp b/nano/qt/qt.cpp index 457d57c781..2c8b6cfb1b 100644 --- a/nano/qt/qt.cpp +++ b/nano/qt/qt.cpp @@ -1,7 +1,6 @@ #include #include -#include #include #include @@ -513,7 +512,7 @@ namespace class short_text_visitor : public nano::block_visitor { public: - short_text_visitor (nano::transaction const & transaction_a, nano::ledger & ledger_a) : + short_text_visitor (nano::store::transaction const & transaction_a, nano::ledger & ledger_a) : transaction (transaction_a), ledger (ledger_a) { @@ -606,7 +605,7 @@ class short_text_visitor : public nano::block_visitor amount = balance - previous_balance; } } - nano::transaction const & transaction; + nano::store::transaction const & transaction; nano::ledger & ledger; std::string type; nano::uint128_t amount; @@ -845,7 +844,7 @@ void nano_qt::stats_viewer::refresh_stats () if (json) { // Format the stat data to make totals and values easier to read - BOOST_FOREACH (boost::property_tree::ptree::value_type const & child, json->get_child ("entries")) + for (boost::property_tree::ptree::value_type const & child : json->get_child ("entries")) { auto time = child.second.get ("time"); auto type = child.second.get ("type"); diff --git a/nano/qt_test/qt.cpp b/nano/qt_test/qt.cpp index 97e9b911a8..8e3dea40b7 100644 --- a/nano/qt_test/qt.cpp +++ b/nano/qt_test/qt.cpp @@ -1,3 +1,4 @@ +#include #include #include #include diff --git a/nano/rpc/rpc_request_processor.cpp b/nano/rpc/rpc_request_processor.cpp index 964bcd2213..d7315890c4 100644 --- a/nano/rpc/rpc_request_processor.cpp +++ b/nano/rpc/rpc_request_processor.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #include #include diff --git a/nano/rpc_test/common.cpp b/nano/rpc_test/common.cpp index 26629cc4a7..dee28c8f73 100644 --- a/nano/rpc_test/common.cpp +++ b/nano/rpc_test/common.cpp @@ -102,7 +102,7 @@ std::shared_ptr nano::test::add_ipc_enabled_node (nano::test::system return add_ipc_enabled_node (system, node_config); } -void nano::test::reset_confirmation_height (nano::store & store, nano::account const & account) +void nano::test::reset_confirmation_height (nano::store::component & store, nano::account const & account) { auto transaction = store.tx_begin_write (); nano::confirmation_height_info confirmation_height_info; diff --git a/nano/rpc_test/common.hpp b/nano/rpc_test/common.hpp index bffc8ac8ea..90d03129e9 100644 --- a/nano/rpc_test/common.hpp +++ b/nano/rpc_test/common.hpp @@ -19,7 +19,10 @@ class node_flags; class node_rpc_config; class public_key; class rpc; -class store; +namespace store +{ + class component; +} using account = public_key; namespace ipc @@ -57,7 +60,7 @@ namespace test std::shared_ptr add_ipc_enabled_node (nano::test::system & system, nano::node_config & node_config, nano::node_flags const & node_flags); std::shared_ptr add_ipc_enabled_node (nano::test::system & system, nano::node_config & node_config); std::shared_ptr add_ipc_enabled_node (nano::test::system & system); - void reset_confirmation_height (nano::store & store, nano::account const & account); + void reset_confirmation_height (nano::store::component & store, nano::account const & account); void wait_response_impl (nano::test::system & system, rpc_context const & rpc_ctx, boost::property_tree::ptree & request, std::chrono::duration const & time, boost::property_tree::ptree & response_json); boost::property_tree::ptree wait_response (nano::test::system & system, rpc_context const & rpc_ctx, boost::property_tree::ptree & request, std::chrono::duration const & time = 5s); bool check_block_response_count (nano::test::system & system, rpc_context const & rpc_ctx, boost::property_tree::ptree & request, uint64_t size_count); diff --git a/nano/rpc_test/rpc.cpp b/nano/rpc_test/rpc.cpp index a48fe5454f..59b83a9fe0 100644 --- a/nano/rpc_test/rpc.cpp +++ b/nano/rpc_test/rpc.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include diff --git a/nano/secure/CMakeLists.txt b/nano/secure/CMakeLists.txt index 1948b1238f..b294503712 100644 --- a/nano/secure/CMakeLists.txt +++ b/nano/secure/CMakeLists.txt @@ -39,8 +39,6 @@ add_library( ${PLATFORM_SECURE_SOURCE} ${CMAKE_BINARY_DIR}/bootstrap_weights_live.cpp ${CMAKE_BINARY_DIR}/bootstrap_weights_beta.cpp - store.hpp - store.cpp common.hpp common.cpp ledger.hpp @@ -49,8 +47,6 @@ add_library( network_filter.cpp utility.hpp utility.cpp - versioning.hpp - versioning.cpp working.hpp) target_link_libraries( @@ -58,7 +54,6 @@ target_link_libraries( nano_lib ed25519 crypto_lib - lmdb Boost::iostreams Boost::system Boost::filesystem) diff --git a/nano/secure/common.cpp b/nano/secure/common.cpp index d5be9a6597..c26b32f893 100644 --- a/nano/secure/common.cpp +++ b/nano/secure/common.cpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include diff --git a/nano/secure/ledger.cpp b/nano/secure/ledger.cpp index 94da1d0d39..a42d29401a 100644 --- a/nano/secure/ledger.cpp +++ b/nano/secure/ledger.cpp @@ -2,9 +2,22 @@ #include #include #include +#include #include #include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include #include @@ -16,7 +29,7 @@ namespace class rollback_visitor : public nano::block_visitor { public: - rollback_visitor (nano::write_transaction const & transaction_a, nano::ledger & ledger_a, std::vector> & list_a) : + rollback_visitor (nano::store::write_transaction const & transaction_a, nano::ledger & ledger_a, std::vector> & list_a) : transaction (transaction_a), ledger (ledger_a), list (list_a) @@ -152,7 +165,7 @@ class rollback_visitor : public nano::block_visitor } debug_assert (!error); - auto previous_version (ledger.store.block.version (transaction, block_a.hashables.previous)); + auto previous_version (ledger.version (transaction, block_a.hashables.previous)); nano::account_info new_info (block_a.hashables.previous, representative, info->open_block, balance, nano::seconds_since_epoch (), info->block_count - 1, previous_version); ledger.update_account (transaction, block_a.hashables.account, *info, new_info); @@ -171,7 +184,7 @@ class rollback_visitor : public nano::block_visitor } ledger.store.block.del (transaction, hash); } - nano::write_transaction const & transaction; + nano::store::write_transaction const & transaction; nano::ledger & ledger; std::vector> & list; bool error{ false }; @@ -180,7 +193,7 @@ class rollback_visitor : public nano::block_visitor class ledger_processor : public nano::mutable_block_visitor { public: - ledger_processor (nano::ledger &, nano::write_transaction const &); + ledger_processor (nano::ledger &, nano::store::write_transaction const &); virtual ~ledger_processor () = default; void send_block (nano::send_block &) override; void receive_block (nano::receive_block &) override; @@ -190,7 +203,7 @@ class ledger_processor : public nano::mutable_block_visitor void state_block_impl (nano::state_block &); void epoch_block_impl (nano::state_block &); nano::ledger & ledger; - nano::write_transaction const & transaction; + nano::store::write_transaction const & transaction; nano::process_return result; private: @@ -671,14 +684,77 @@ void ledger_processor::open_block (nano::open_block & block_a) } } -ledger_processor::ledger_processor (nano::ledger & ledger_a, nano::write_transaction const & transaction_a) : +ledger_processor::ledger_processor (nano::ledger & ledger_a, nano::store::write_transaction const & transaction_a) : ledger (ledger_a), transaction (transaction_a) { } + +/** + * Determine the representative for this block + */ +class representative_visitor final : public nano::block_visitor +{ +public: + representative_visitor (nano::store::transaction const & transaction_a, nano::store::component & store_a); + ~representative_visitor () = default; + void compute (nano::block_hash const & hash_a); + void send_block (nano::send_block const & block_a) override; + void receive_block (nano::receive_block const & block_a) override; + void open_block (nano::open_block const & block_a) override; + void change_block (nano::change_block const & block_a) override; + void state_block (nano::state_block const & block_a) override; + nano::store::transaction const & transaction; + nano::store::component & store; + nano::block_hash current; + nano::block_hash result; +}; + +representative_visitor::representative_visitor (nano::store::transaction const & transaction_a, nano::store::component & store_a) : + transaction (transaction_a), + store (store_a), + result (0) +{ +} + +void representative_visitor::compute (nano::block_hash const & hash_a) +{ + current = hash_a; + while (result.is_zero ()) + { + auto block (store.block.get (transaction, current)); + debug_assert (block != nullptr); + block->visit (*this); + } +} + +void representative_visitor::send_block (nano::send_block const & block_a) +{ + current = block_a.previous (); +} + +void representative_visitor::receive_block (nano::receive_block const & block_a) +{ + current = block_a.previous (); +} + +void representative_visitor::open_block (nano::open_block const & block_a) +{ + result = block_a.hash (); +} + +void representative_visitor::change_block (nano::change_block const & block_a) +{ + result = block_a.hash (); +} + +void representative_visitor::state_block (nano::state_block const & block_a) +{ + result = block_a.hash (); +} } // namespace -nano::ledger::ledger (nano::store & store_a, nano::stats & stat_a, nano::ledger_constants & constants, nano::generate_cache const & generate_cache_a) : +nano::ledger::ledger (nano::store::component & store_a, nano::stats & stat_a, nano::ledger_constants & constants, nano::generate_cache const & generate_cache_a) : constants{ constants }, store{ store_a }, stats{ stat_a }, @@ -695,7 +771,7 @@ void nano::ledger::initialize (nano::generate_cache const & generate_cache_a) if (generate_cache_a.reps || generate_cache_a.account_count || generate_cache_a.block_count) { store.account.for_each_par ( - [this] (nano::read_transaction const & /*unused*/, nano::store_iterator i, nano::store_iterator n) { + [this] (store::read_transaction const & /*unused*/, store::iterator i, store::iterator n) { uint64_t block_count_l{ 0 }; uint64_t account_count_l{ 0 }; decltype (this->cache.rep_weights) rep_weights_l; @@ -715,7 +791,7 @@ void nano::ledger::initialize (nano::generate_cache const & generate_cache_a) if (generate_cache_a.cemented_count) { store.confirmation_height.for_each_par ( - [this] (nano::read_transaction const & /*unused*/, nano::store_iterator i, nano::store_iterator n) { + [this] (store::read_transaction const & /*unused*/, store::iterator i, store::iterator n) { uint64_t cemented_count_l (0); for (; i != n; ++i) { @@ -736,13 +812,41 @@ void nano::ledger::initialize (nano::generate_cache const & generate_cache_a) } } +nano::uint128_t nano::ledger::balance (nano::block const & block) +{ + nano::uint128_t result; + switch (block.type ()) + { + case nano::block_type::open: + case nano::block_type::receive: + case nano::block_type::change: + result = block.sideband ().balance.number (); + break; + case nano::block_type::send: + case nano::block_type::state: + result = block.balance ().number (); + break; + case nano::block_type::invalid: + case nano::block_type::not_a_block: + release_assert (false); + break; + } + return result; +} + // Balance for account containing hash -nano::uint128_t nano::ledger::balance (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const +nano::uint128_t nano::ledger::balance (store::transaction const & transaction, nano::block_hash const & hash) const { - return hash_a.is_zero () ? 0 : store.block.balance (transaction_a, hash_a); + if (hash.is_zero ()) + { + return 0; + } + auto block = store.block.get (transaction, hash); + debug_assert (block != nullptr); + return balance (*block); } -nano::uint128_t nano::ledger::balance_safe (nano::transaction const & transaction_a, nano::block_hash const & hash_a, bool & error_a) const +nano::uint128_t nano::ledger::balance_safe (store::transaction const & transaction_a, nano::block_hash const & hash_a, bool & error_a) const { nano::uint128_t result (0); if (pruning && !hash_a.is_zero () && !store.block.exists (transaction_a, hash_a)) @@ -758,7 +862,7 @@ nano::uint128_t nano::ledger::balance_safe (nano::transaction const & transactio } // Balance for an account by account number -nano::uint128_t nano::ledger::account_balance (nano::transaction const & transaction_a, nano::account const & account_a, bool only_confirmed_a) +nano::uint128_t nano::ledger::account_balance (store::transaction const & transaction_a, nano::account const & account_a, bool only_confirmed_a) { nano::uint128_t result (0); if (only_confirmed_a) @@ -780,7 +884,7 @@ nano::uint128_t nano::ledger::account_balance (nano::transaction const & transac return result; } -nano::uint128_t nano::ledger::account_receivable (nano::transaction const & transaction_a, nano::account const & account_a, bool only_confirmed_a) +nano::uint128_t nano::ledger::account_receivable (store::transaction const & transaction_a, nano::account const & account_a, bool only_confirmed_a) { nano::uint128_t result (0); nano::account end (account_a.number () + 1); @@ -802,7 +906,7 @@ nano::uint128_t nano::ledger::account_receivable (nano::transaction const & tran return result; } -std::optional nano::ledger::pending_info (nano::transaction const & transaction, nano::pending_key const & key) const +std::optional nano::ledger::pending_info (store::transaction const & transaction, nano::pending_key const & key) const { nano::pending_info result; if (!store.pending.get (transaction, key, result)) @@ -812,7 +916,7 @@ std::optional nano::ledger::pending_info (nano::transaction return std::nullopt; } -nano::process_return nano::ledger::process (nano::write_transaction const & transaction_a, nano::block & block_a) +nano::process_return nano::ledger::process (store::write_transaction const & transaction_a, nano::block & block_a) { debug_assert (!constants.work.validate_entry (block_a) || constants.genesis == nano::dev::genesis); ledger_processor processor (*this, transaction_a); @@ -824,14 +928,14 @@ nano::process_return nano::ledger::process (nano::write_transaction const & tran return processor.result; } -nano::block_hash nano::ledger::representative (nano::transaction const & transaction_a, nano::block_hash const & hash_a) +nano::block_hash nano::ledger::representative (store::transaction const & transaction_a, nano::block_hash const & hash_a) { auto result (representative_calculated (transaction_a, hash_a)); debug_assert (result.is_zero () || store.block.exists (transaction_a, result)); return result; } -nano::block_hash nano::ledger::representative_calculated (nano::transaction const & transaction_a, nano::block_hash const & hash_a) +nano::block_hash nano::ledger::representative_calculated (store::transaction const & transaction_a, nano::block_hash const & hash_a) { representative_visitor visitor (transaction_a, store); visitor.compute (hash_a); @@ -843,7 +947,7 @@ bool nano::ledger::block_or_pruned_exists (nano::block_hash const & hash_a) cons return block_or_pruned_exists (store.tx_begin_read (), hash_a); } -bool nano::ledger::block_or_pruned_exists (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const +bool nano::ledger::block_or_pruned_exists (store::transaction const & transaction_a, nano::block_hash const & hash_a) const { if (store.pruned.exists (transaction_a, hash_a)) { @@ -852,7 +956,7 @@ bool nano::ledger::block_or_pruned_exists (nano::transaction const & transaction return store.block.exists (transaction_a, hash_a); } -bool nano::ledger::root_exists (nano::transaction const & transaction_a, nano::root const & root_a) +bool nano::ledger::root_exists (store::transaction const & transaction_a, nano::root const & root_a) { return store.block.exists (transaction_a, root_a.as_block_hash ()) || store.account.exists (transaction_a, root_a.as_account ()); } @@ -874,7 +978,7 @@ std::string nano::ledger::block_text (nano::block_hash const & hash_a) return result; } -bool nano::ledger::is_send (nano::transaction const & transaction_a, nano::block const & block_a) const +bool nano::ledger::is_send (store::transaction const & transaction_a, nano::block const & block_a) const { if (block_a.type () != nano::block_type::state) { @@ -906,7 +1010,7 @@ bool nano::ledger::is_send (nano::transaction const & transaction_a, nano::block return result; } -nano::account const & nano::ledger::block_destination (nano::transaction const & transaction_a, nano::block const & block_a) +nano::account const & nano::ledger::block_destination (store::transaction const & transaction_a, nano::block const & block_a) { nano::send_block const * send_block (dynamic_cast (&block_a)); nano::state_block const * state_block (dynamic_cast (&block_a)); @@ -922,7 +1026,7 @@ nano::account const & nano::ledger::block_destination (nano::transaction const & return nano::account::null (); } -nano::block_hash nano::ledger::block_source (nano::transaction const & transaction_a, nano::block const & block_a) +nano::block_hash nano::ledger::block_source (store::transaction const & transaction_a, nano::block const & block_a) { /* * block_source() requires that the previous block of the block @@ -942,7 +1046,7 @@ nano::block_hash nano::ledger::block_source (nano::transaction const & transacti return result; } -std::pair nano::ledger::hash_root_random (nano::transaction const & transaction_a) const +std::pair nano::ledger::hash_root_random (store::transaction const & transaction_a) const { nano::block_hash hash (0); nano::root root (0); @@ -993,11 +1097,11 @@ nano::uint128_t nano::ledger::weight (nano::account const & account_a) } // Rollback blocks until `block_a' doesn't exist or it tries to penetrate the confirmation height -bool nano::ledger::rollback (nano::write_transaction const & transaction_a, nano::block_hash const & block_a, std::vector> & list_a) +bool nano::ledger::rollback (store::write_transaction const & transaction_a, nano::block_hash const & block_a, std::vector> & list_a) { debug_assert (store.block.exists (transaction_a, block_a)); auto account_l (account (transaction_a, block_a)); - auto block_account_height (store.block.account_height (transaction_a, block_a)); + auto block_account_height (height (transaction_a, block_a)); rollback_visitor rollback (transaction_a, *this, list_a); auto error (false); while (!error && store.block.exists (transaction_a, block_a)) @@ -1025,29 +1129,43 @@ bool nano::ledger::rollback (nano::write_transaction const & transaction_a, nano return error; } -bool nano::ledger::rollback (nano::write_transaction const & transaction_a, nano::block_hash const & block_a) +bool nano::ledger::rollback (store::write_transaction const & transaction_a, nano::block_hash const & block_a) { std::vector> rollback_list; return rollback (transaction_a, block_a, rollback_list); } -nano::account nano::ledger::account (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const +nano::account nano::ledger::account (nano::block const & block) const +{ + debug_assert (block.has_sideband ()); + nano::account result (block.account ()); + if (result.is_zero ()) + { + result = block.sideband ().account; + } + debug_assert (!result.is_zero ()); + return result; +} + +nano::account nano::ledger::account (store::transaction const & transaction, nano::block_hash const & hash) const { - return store.block.account (transaction_a, hash_a); + auto block = store.block.get (transaction, hash); + debug_assert (block != nullptr); + return account (*block); } -nano::account nano::ledger::account_safe (nano::transaction const & transaction_a, nano::block_hash const & hash_a, bool & error_a) const +nano::account nano::ledger::account_safe (store::transaction const & transaction_a, nano::block_hash const & hash_a, bool & error_a) const { if (!pruning) { - return store.block.account (transaction_a, hash_a); + return account (transaction_a, hash_a); } else { auto block (store.block.get (transaction_a, hash_a)); if (block != nullptr) { - return store.block.account_calculated (*block); + return account (*block); } else { @@ -1057,12 +1175,12 @@ nano::account nano::ledger::account_safe (nano::transaction const & transaction_ } } -nano::account nano::ledger::account_safe (const nano::transaction & transaction, const nano::block_hash & hash) const +nano::account nano::ledger::account_safe (store::transaction const & transaction, nano::block_hash const & hash) const { auto block = store.block.get (transaction, hash); if (block) { - return store.block.account_calculated (*block); + return account (*block); } else { @@ -1070,19 +1188,19 @@ nano::account nano::ledger::account_safe (const nano::transaction & transaction, } } -std::optional nano::ledger::account_info (nano::transaction const & transaction, nano::account const & account) const +std::optional nano::ledger::account_info (store::transaction const & transaction, nano::account const & account) const { return store.account.get (transaction, account); } // Return amount decrease or increase for block -nano::uint128_t nano::ledger::amount (nano::transaction const & transaction_a, nano::account const & account_a) +nano::uint128_t nano::ledger::amount (store::transaction const & transaction_a, nano::account const & account_a) { release_assert (account_a == constants.genesis->account ()); return nano::dev::constants.genesis_amount; } -nano::uint128_t nano::ledger::amount (nano::transaction const & transaction_a, nano::block_hash const & hash_a) +nano::uint128_t nano::ledger::amount (store::transaction const & transaction_a, nano::block_hash const & hash_a) { auto block (store.block.get (transaction_a, hash_a)); auto block_balance (balance (transaction_a, hash_a)); @@ -1090,7 +1208,7 @@ nano::uint128_t nano::ledger::amount (nano::transaction const & transaction_a, n return block_balance > previous_balance ? block_balance - previous_balance : previous_balance - block_balance; } -nano::uint128_t nano::ledger::amount_safe (nano::transaction const & transaction_a, nano::block_hash const & hash_a, bool & error_a) const +nano::uint128_t nano::ledger::amount_safe (store::transaction const & transaction_a, nano::block_hash const & hash_a, bool & error_a) const { auto block (store.block.get (transaction_a, hash_a)); debug_assert (block); @@ -1101,14 +1219,14 @@ nano::uint128_t nano::ledger::amount_safe (nano::transaction const & transaction } // Return latest block for account -nano::block_hash nano::ledger::latest (nano::transaction const & transaction_a, nano::account const & account_a) +nano::block_hash nano::ledger::latest (store::transaction const & transaction_a, nano::account const & account_a) { auto info = account_info (transaction_a, account_a); return !info ? 0 : info->head; } // Return latest root for account, account number if there are no blocks for this account. -nano::root nano::ledger::latest_root (nano::transaction const & transaction_a, nano::account const & account_a) +nano::root nano::ledger::latest_root (store::transaction const & transaction_a, nano::account const & account_a) { auto info = account_info (transaction_a, account_a); if (!info) @@ -1134,7 +1252,7 @@ void nano::ledger::dump_account_chain (nano::account const & account_a, std::ost } } -bool nano::ledger::could_fit (nano::transaction const & transaction_a, nano::block const & block_a) const +bool nano::ledger::could_fit (store::transaction const & transaction_a, nano::block const & block_a) const { auto dependencies (dependent_blocks (transaction_a, block_a)); return std::all_of (dependencies.begin (), dependencies.end (), [this, &transaction_a] (nano::block_hash const & hash_a) { @@ -1142,7 +1260,7 @@ bool nano::ledger::could_fit (nano::transaction const & transaction_a, nano::blo }); } -bool nano::ledger::dependents_confirmed (nano::transaction const & transaction_a, nano::block const & block_a) const +bool nano::ledger::dependents_confirmed (store::transaction const & transaction_a, nano::block const & block_a) const { auto dependencies (dependent_blocks (transaction_a, block_a)); return std::all_of (dependencies.begin (), dependencies.end (), [this, &transaction_a] (nano::block_hash const & hash_a) { @@ -1163,7 +1281,7 @@ bool nano::ledger::is_epoch_link (nano::link const & link_a) const class dependent_block_visitor : public nano::block_visitor { public: - dependent_block_visitor (nano::ledger const & ledger_a, nano::transaction const & transaction_a) : + dependent_block_visitor (nano::ledger const & ledger_a, nano::store::transaction const & transaction_a) : ledger (ledger_a), transaction (transaction_a), result ({ 0, 0 }) @@ -1200,11 +1318,11 @@ class dependent_block_visitor : public nano::block_visitor } } nano::ledger const & ledger; - nano::transaction const & transaction; + nano::store::transaction const & transaction; std::array result; }; -std::array nano::ledger::dependent_blocks (nano::transaction const & transaction_a, nano::block const & block_a) const +std::array nano::ledger::dependent_blocks (store::transaction const & transaction_a, nano::block const & block_a) const { dependent_block_visitor visitor (*this, transaction_a); block_a.visit (visitor); @@ -1215,7 +1333,7 @@ std::array nano::ledger::dependent_blocks (nano::transactio * The send block hash is not checked in any way, it is assumed to be correct. * @return Return the receive block on success and null on failure */ -std::shared_ptr nano::ledger::find_receive_block_by_send_hash (nano::transaction const & transaction, nano::account const & destination, nano::block_hash const & send_block_hash) +std::shared_ptr nano::ledger::find_receive_block_by_send_hash (store::transaction const & transaction, nano::account const & destination, nano::block_hash const & send_block_hash) { std::shared_ptr result; debug_assert (send_block_hash != 0); @@ -1269,7 +1387,7 @@ nano::link const & nano::ledger::epoch_link (nano::epoch epoch_a) const return constants.epochs.link (epoch_a); } -void nano::ledger::update_account (nano::write_transaction const & transaction_a, nano::account const & account_a, nano::account_info const & old_a, nano::account_info const & new_a) +void nano::ledger::update_account (store::write_transaction const & transaction_a, nano::account const & account_a, nano::account_info const & old_a, nano::account_info const & new_a) { if (!new_a.head.is_zero ()) { @@ -1293,7 +1411,7 @@ void nano::ledger::update_account (nano::write_transaction const & transaction_a } } -std::shared_ptr nano::ledger::successor (nano::transaction const & transaction_a, nano::qualified_root const & root_a) +std::shared_ptr nano::ledger::successor (store::transaction const & transaction_a, nano::qualified_root const & root_a) { nano::block_hash successor (0); auto get_from_previous = false; @@ -1327,7 +1445,7 @@ std::shared_ptr nano::ledger::successor (nano::transaction const & return result; } -std::shared_ptr nano::ledger::forked_block (nano::transaction const & transaction_a, nano::block const & block_a) +std::shared_ptr nano::ledger::forked_block (store::transaction const & transaction_a, nano::block const & block_a) { debug_assert (!store.block.exists (transaction_a, block_a.hash ())); auto root (block_a.root ()); @@ -1343,7 +1461,7 @@ std::shared_ptr nano::ledger::forked_block (nano::transaction const return result; } -std::shared_ptr nano::ledger::head_block (nano::transaction const & transaction, nano::account const & account) +std::shared_ptr nano::ledger::head_block (store::transaction const & transaction, nano::account const & account) { auto info = store.account.get (transaction, account); if (info) @@ -1353,7 +1471,7 @@ std::shared_ptr nano::ledger::head_block (nano::transaction const & return nullptr; } -bool nano::ledger::block_confirmed (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const +bool nano::ledger::block_confirmed (store::transaction const & transaction_a, nano::block_hash const & hash_a) const { if (store.pruned.exists (transaction_a, hash_a)) { @@ -1370,7 +1488,7 @@ bool nano::ledger::block_confirmed (nano::transaction const & transaction_a, nan return false; } -uint64_t nano::ledger::pruning_action (nano::write_transaction & transaction_a, nano::block_hash const & hash_a, uint64_t const batch_size_a) +uint64_t nano::ledger::pruning_action (store::write_transaction & transaction_a, nano::block_hash const & hash_a, uint64_t const batch_size_a) { uint64_t pruned_count (0); nano::block_hash hash (hash_a); @@ -1408,7 +1526,7 @@ std::multimap> nano::ledger::unc nano::locked>> result; using result_t = decltype (result)::value_type; - store.account.for_each_par ([this, &result] (nano::read_transaction const & transaction_a, nano::store_iterator i, nano::store_iterator n) { + store.account.for_each_par ([this, &result] (store::read_transaction const & transaction_a, store::iterator i, store::iterator n) { result_t unconfirmed_frontiers_l; for (; i != n; ++i) { @@ -1453,7 +1571,7 @@ bool nano::ledger::migrate_lmdb_to_rocksdb (boost::filesystem::path const & data if (!rocksdb_store->init_error ()) { store.block.for_each_par ( - [&rocksdb_store] (nano::read_transaction const & /*unused*/, auto i, auto n) { + [&rocksdb_store] (store::read_transaction const & /*unused*/, auto i, auto n) { for (; i != n; ++i) { auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::blocks })); @@ -1469,7 +1587,7 @@ bool nano::ledger::migrate_lmdb_to_rocksdb (boost::filesystem::path const & data }); store.pending.for_each_par ( - [&rocksdb_store] (nano::read_transaction const & /*unused*/, auto i, auto n) { + [&rocksdb_store] (store::read_transaction const & /*unused*/, auto i, auto n) { for (; i != n; ++i) { auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::pending })); @@ -1478,7 +1596,7 @@ bool nano::ledger::migrate_lmdb_to_rocksdb (boost::filesystem::path const & data }); store.confirmation_height.for_each_par ( - [&rocksdb_store] (nano::read_transaction const & /*unused*/, auto i, auto n) { + [&rocksdb_store] (store::read_transaction const & /*unused*/, auto i, auto n) { for (; i != n; ++i) { auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::confirmation_height })); @@ -1487,7 +1605,7 @@ bool nano::ledger::migrate_lmdb_to_rocksdb (boost::filesystem::path const & data }); store.account.for_each_par ( - [&rocksdb_store] (nano::read_transaction const & /*unused*/, auto i, auto n) { + [&rocksdb_store] (store::read_transaction const & /*unused*/, auto i, auto n) { for (; i != n; ++i) { auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::accounts })); @@ -1496,7 +1614,7 @@ bool nano::ledger::migrate_lmdb_to_rocksdb (boost::filesystem::path const & data }); store.frontier.for_each_par ( - [&rocksdb_store] (nano::read_transaction const & /*unused*/, auto i, auto n) { + [&rocksdb_store] (store::read_transaction const & /*unused*/, auto i, auto n) { for (; i != n; ++i) { auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::frontiers })); @@ -1505,7 +1623,7 @@ bool nano::ledger::migrate_lmdb_to_rocksdb (boost::filesystem::path const & data }); store.pruned.for_each_par ( - [&rocksdb_store] (nano::read_transaction const & /*unused*/, auto i, auto n) { + [&rocksdb_store] (store::read_transaction const & /*unused*/, auto i, auto n) { for (; i != n; ++i) { auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::pruned })); @@ -1514,7 +1632,7 @@ bool nano::ledger::migrate_lmdb_to_rocksdb (boost::filesystem::path const & data }); store.final_vote.for_each_par ( - [&rocksdb_store] (nano::read_transaction const & /*unused*/, auto i, auto n) { + [&rocksdb_store] (store::read_transaction const & /*unused*/, auto i, auto n) { for (; i != n; ++i) { auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::final_votes })); @@ -1571,6 +1689,32 @@ bool nano::ledger::bootstrap_weight_reached () const return cache.block_count >= bootstrap_weight_max_blocks; } +nano::epoch nano::ledger::version (nano::block const & block) +{ + if (block.type () == nano::block_type::state) + { + return block.sideband ().details.epoch; + } + + return nano::epoch::epoch_0; +} + +nano::epoch nano::ledger::version (store::transaction const & transaction, nano::block_hash const & hash) const +{ + auto block = store.block.get (transaction, hash); + if (block == nullptr) + { + return nano::epoch::epoch_0; + } + return version (*block); +} + +uint64_t nano::ledger::height (store::transaction const & transaction, nano::block_hash const & hash) const +{ + auto block = store.block.get (transaction, hash); + return block->sideband ().height; +} + nano::uncemented_info::uncemented_info (nano::block_hash const & cemented_frontier, nano::block_hash const & frontier, nano::account const & account) : cemented_frontier (cemented_frontier), frontier (frontier), account (account) { diff --git a/nano/secure/ledger.hpp b/nano/secure/ledger.hpp index 6284ee0434..d66e20241e 100644 --- a/nano/secure/ledger.hpp +++ b/nano/secure/ledger.hpp @@ -6,11 +6,16 @@ #include +namespace nano::store +{ +class component; +class transaction; +class write_transaction; +} + namespace nano { -class store; class stats; -class write_transaction; // map of vote weight per block, ordered greater first using tally_t = std::map, std::greater>; @@ -27,67 +32,72 @@ class uncemented_info class ledger final { public: - ledger (nano::store &, nano::stats &, nano::ledger_constants & constants, nano::generate_cache const & = nano::generate_cache ()); + ledger (nano::store::component &, nano::stats &, nano::ledger_constants & constants, nano::generate_cache const & = nano::generate_cache ()); /** * Return account containing hash, expects that block hash exists in ledger */ - nano::account account (nano::transaction const &, nano::block_hash const &) const; - std::optional account_info (nano::transaction const & transaction, nano::account const & account) const; + nano::account account (nano::block const & block) const; + nano::account account (store::transaction const &, nano::block_hash const &) const; + std::optional account_info (store::transaction const & transaction, nano::account const & account) const; /** * For non-prunning nodes same as `ledger::account()` * For prunning nodes ensures that block hash exists, otherwise returns zero account */ - nano::account account_safe (nano::transaction const &, nano::block_hash const &, bool &) const; + nano::account account_safe (store::transaction const &, nano::block_hash const &, bool &) const; /** * Return account containing hash, returns zero account if account can not be found */ - nano::account account_safe (nano::transaction const &, nano::block_hash const &) const; - nano::uint128_t amount (nano::transaction const &, nano::account const &); - nano::uint128_t amount (nano::transaction const &, nano::block_hash const &); + nano::account account_safe (store::transaction const &, nano::block_hash const &) const; + nano::uint128_t amount (store::transaction const &, nano::account const &); + nano::uint128_t amount (store::transaction const &, nano::block_hash const &); /** Safe for previous block, but block hash_a must exist */ - nano::uint128_t amount_safe (nano::transaction const &, nano::block_hash const & hash_a, bool &) const; - nano::uint128_t balance (nano::transaction const &, nano::block_hash const &) const; - nano::uint128_t balance_safe (nano::transaction const &, nano::block_hash const &, bool &) const; - nano::uint128_t account_balance (nano::transaction const &, nano::account const &, bool = false); - nano::uint128_t account_receivable (nano::transaction const &, nano::account const &, bool = false); + nano::uint128_t amount_safe (store::transaction const &, nano::block_hash const & hash_a, bool &) const; + static nano::uint128_t balance (nano::block const & block); + nano::uint128_t balance (store::transaction const &, nano::block_hash const &) const; + nano::uint128_t balance_safe (store::transaction const &, nano::block_hash const &, bool &) const; + nano::uint128_t account_balance (store::transaction const &, nano::account const &, bool = false); + nano::uint128_t account_receivable (store::transaction const &, nano::account const &, bool = false); nano::uint128_t weight (nano::account const &); - std::shared_ptr successor (nano::transaction const &, nano::qualified_root const &); - std::shared_ptr forked_block (nano::transaction const &, nano::block const &); - std::shared_ptr head_block (nano::transaction const &, nano::account const &); - bool block_confirmed (nano::transaction const &, nano::block_hash const &) const; - nano::block_hash latest (nano::transaction const &, nano::account const &); - nano::root latest_root (nano::transaction const &, nano::account const &); - nano::block_hash representative (nano::transaction const &, nano::block_hash const &); - nano::block_hash representative_calculated (nano::transaction const &, nano::block_hash const &); + std::shared_ptr successor (store::transaction const &, nano::qualified_root const &); + std::shared_ptr forked_block (store::transaction const &, nano::block const &); + std::shared_ptr head_block (store::transaction const &, nano::account const &); + bool block_confirmed (store::transaction const &, nano::block_hash const &) const; + nano::block_hash latest (store::transaction const &, nano::account const &); + nano::root latest_root (store::transaction const &, nano::account const &); + nano::block_hash representative (store::transaction const &, nano::block_hash const &); + nano::block_hash representative_calculated (store::transaction const &, nano::block_hash const &); bool block_or_pruned_exists (nano::block_hash const &) const; - bool block_or_pruned_exists (nano::transaction const &, nano::block_hash const &) const; - bool root_exists (nano::transaction const &, nano::root const &); + bool block_or_pruned_exists (store::transaction const &, nano::block_hash const &) const; + bool root_exists (store::transaction const &, nano::root const &); std::string block_text (char const *); std::string block_text (nano::block_hash const &); - bool is_send (nano::transaction const &, nano::block const &) const; - nano::account const & block_destination (nano::transaction const &, nano::block const &); - nano::block_hash block_source (nano::transaction const &, nano::block const &); - std::pair hash_root_random (nano::transaction const &) const; - std::optional pending_info (nano::transaction const & transaction, nano::pending_key const & key) const; - nano::process_return process (nano::write_transaction const &, nano::block &); - bool rollback (nano::write_transaction const &, nano::block_hash const &, std::vector> &); - bool rollback (nano::write_transaction const &, nano::block_hash const &); - void update_account (nano::write_transaction const &, nano::account const &, nano::account_info const &, nano::account_info const &); - uint64_t pruning_action (nano::write_transaction &, nano::block_hash const &, uint64_t const); + bool is_send (store::transaction const &, nano::block const &) const; + nano::account const & block_destination (store::transaction const &, nano::block const &); + nano::block_hash block_source (store::transaction const &, nano::block const &); + std::pair hash_root_random (store::transaction const &) const; + std::optional pending_info (store::transaction const & transaction, nano::pending_key const & key) const; + nano::process_return process (store::write_transaction const &, nano::block &); + bool rollback (store::write_transaction const &, nano::block_hash const &, std::vector> &); + bool rollback (store::write_transaction const &, nano::block_hash const &); + void update_account (store::write_transaction const &, nano::account const &, nano::account_info const &, nano::account_info const &); + uint64_t pruning_action (store::write_transaction &, nano::block_hash const &, uint64_t const); void dump_account_chain (nano::account const &, std::ostream & = std::cout); - bool could_fit (nano::transaction const &, nano::block const &) const; - bool dependents_confirmed (nano::transaction const &, nano::block const &) const; + bool could_fit (store::transaction const &, nano::block const &) const; + bool dependents_confirmed (store::transaction const &, nano::block const &) const; bool is_epoch_link (nano::link const &) const; - std::array dependent_blocks (nano::transaction const &, nano::block const &) const; - std::shared_ptr find_receive_block_by_send_hash (nano::transaction const & transaction, nano::account const & destination, nano::block_hash const & send_block_hash); + std::array dependent_blocks (store::transaction const &, nano::block const &) const; + std::shared_ptr find_receive_block_by_send_hash (store::transaction const & transaction, nano::account const & destination, nano::block_hash const & send_block_hash); nano::account const & epoch_signer (nano::link const &) const; nano::link const & epoch_link (nano::epoch) const; std::multimap> unconfirmed_frontiers () const; bool migrate_lmdb_to_rocksdb (boost::filesystem::path const &) const; bool bootstrap_weight_reached () const; + static nano::epoch version (nano::block const & block); + nano::epoch version (store::transaction const & transaction, nano::block_hash const & hash) const; + uint64_t height (store::transaction const & transaction, nano::block_hash const & hash) const; static nano::uint128_t const unit; nano::ledger_constants & constants; - nano::store & store; + nano::store::component & store; nano::ledger_cache cache; nano::stats & stats; std::unordered_map bootstrap_weights; diff --git a/nano/secure/parallel_traversal.hpp b/nano/secure/parallel_traversal.hpp index 3bc261f8ea..cf1a51ac4a 100644 --- a/nano/secure/parallel_traversal.hpp +++ b/nano/secure/parallel_traversal.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include diff --git a/nano/secure/plat/osx/working.mm b/nano/secure/plat/osx/working.mm index 48f8c50a62..28aa1005af 100644 --- a/nano/secure/plat/osx/working.mm +++ b/nano/secure/plat/osx/working.mm @@ -1,5 +1,7 @@ #include +#include + #include namespace nano @@ -12,4 +14,4 @@ [dir_string release]; return result; } -} \ No newline at end of file +} diff --git a/nano/secure/plat/posix/working.cpp b/nano/secure/plat/posix/working.cpp index 1d36b163ba..c962bd774b 100644 --- a/nano/secure/plat/posix/working.cpp +++ b/nano/secure/plat/posix/working.cpp @@ -1,6 +1,8 @@ #include #include +#include + #include #include diff --git a/nano/secure/store.cpp b/nano/secure/store.cpp deleted file mode 100644 index cc3e727660..0000000000 --- a/nano/secure/store.cpp +++ /dev/null @@ -1,182 +0,0 @@ -#include -#include -#include - -nano::representative_visitor::representative_visitor (nano::transaction const & transaction_a, nano::store & store_a) : - transaction (transaction_a), - store (store_a), - result (0) -{ -} - -void nano::representative_visitor::compute (nano::block_hash const & hash_a) -{ - current = hash_a; - while (result.is_zero ()) - { - auto block (store.block.get (transaction, current)); - debug_assert (block != nullptr); - block->visit (*this); - } -} - -void nano::representative_visitor::send_block (nano::send_block const & block_a) -{ - current = block_a.previous (); -} - -void nano::representative_visitor::receive_block (nano::receive_block const & block_a) -{ - current = block_a.previous (); -} - -void nano::representative_visitor::open_block (nano::open_block const & block_a) -{ - result = block_a.hash (); -} - -void nano::representative_visitor::change_block (nano::change_block const & block_a) -{ - result = block_a.hash (); -} - -void nano::representative_visitor::state_block (nano::state_block const & block_a) -{ - result = block_a.hash (); -} - -nano::read_transaction::read_transaction (std::unique_ptr read_transaction_impl) : - impl (std::move (read_transaction_impl)) -{ -} - -void * nano::read_transaction::get_handle () const -{ - return impl->get_handle (); -} - -void nano::read_transaction::reset () const -{ - impl->reset (); -} - -void nano::read_transaction::renew () const -{ - impl->renew (); -} - -void nano::read_transaction::refresh () const -{ - reset (); - renew (); -} - -nano::write_transaction::write_transaction (std::unique_ptr write_transaction_impl) : - impl (std::move (write_transaction_impl)) -{ - /* - * For IO threads, we do not want them to block on creating write transactions. - */ - debug_assert (nano::thread_role::get () != nano::thread_role::name::io); -} - -void * nano::write_transaction::get_handle () const -{ - return impl->get_handle (); -} - -void nano::write_transaction::commit () -{ - impl->commit (); -} - -void nano::write_transaction::renew () -{ - impl->renew (); -} - -void nano::write_transaction::refresh () -{ - impl->commit (); - impl->renew (); -} - -bool nano::write_transaction::contains (nano::tables table_a) const -{ - return impl->contains (table_a); -} - -// clang-format off -nano::store::store ( - nano::block_store & block_store_a, - nano::frontier_store & frontier_store_a, - nano::account_store & account_store_a, - nano::pending_store & pending_store_a, - nano::online_weight_store & online_weight_store_a, - nano::pruned_store & pruned_store_a, - nano::peer_store & peer_store_a, - nano::confirmation_height_store & confirmation_height_store_a, - nano::final_vote_store & final_vote_store_a, - nano::version_store & version_store_a -) : - block (block_store_a), - frontier (frontier_store_a), - account (account_store_a), - pending (pending_store_a), - online_weight (online_weight_store_a), - pruned (pruned_store_a), - peer (peer_store_a), - confirmation_height (confirmation_height_store_a), - final_vote (final_vote_store_a), - version (version_store_a) -{ -} -// clang-format on - -/** - * If using a different store version than the latest then you may need - * to modify some of the objects in the store to be appropriate for the version before an upgrade. - */ -void nano::store::initialize (nano::write_transaction const & transaction_a, nano::ledger_cache & ledger_cache_a, nano::ledger_constants & constants) -{ - debug_assert (constants.genesis->has_sideband ()); - debug_assert (account.begin (transaction_a) == account.end ()); - auto hash_l (constants.genesis->hash ()); - block.put (transaction_a, hash_l, *constants.genesis); - ++ledger_cache_a.block_count; - confirmation_height.put (transaction_a, constants.genesis->account (), nano::confirmation_height_info{ 1, constants.genesis->hash () }); - ++ledger_cache_a.cemented_count; - ledger_cache_a.final_votes_confirmation_canary = (constants.final_votes_canary_account == constants.genesis->account () && 1 >= constants.final_votes_canary_height); - account.put (transaction_a, constants.genesis->account (), { hash_l, constants.genesis->account (), constants.genesis->hash (), std::numeric_limits::max (), nano::seconds_since_epoch (), 1, nano::epoch::epoch_0 }); - ++ledger_cache_a.account_count; - ledger_cache_a.rep_weights.representation_put (constants.genesis->account (), std::numeric_limits::max ()); - frontier.put (transaction_a, hash_l, constants.genesis->account ()); -} - -std::optional nano::account_store::get (const nano::transaction & transaction, const nano::account & account) -{ - nano::account_info info; - bool error = get (transaction, account, info); - if (!error) - { - return info; - } - else - { - return std::nullopt; - } -} - -std::optional nano::confirmation_height_store::get (const nano::transaction & transaction, const nano::account & account) -{ - nano::confirmation_height_info info; - bool error = get (transaction, account, info); - if (!error) - { - return info; - } - else - { - return std::nullopt; - } -} \ No newline at end of file diff --git a/nano/secure/store.hpp b/nano/secure/store.hpp deleted file mode 100644 index 57f0ac0492..0000000000 --- a/nano/secure/store.hpp +++ /dev/null @@ -1,848 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include - -namespace nano -{ -// Move to versioning with a specific version if required for a future upgrade -template -class block_w_sideband_v18 -{ -public: - std::shared_ptr block; - nano::block_sideband_v18 sideband; -}; - -class block_w_sideband -{ -public: - std::shared_ptr block; - nano::block_sideband sideband; -}; - -/** - * Encapsulates database specific container - */ -template -class db_val -{ -public: - db_val (Val const & value_a) : - value (value_a) - { - } - - db_val () : - db_val (0, nullptr) - { - } - - db_val (std::nullptr_t) : - db_val (0, this) - { - } - - db_val (nano::uint128_union const & val_a) : - db_val (sizeof (val_a), const_cast (&val_a)) - { - } - - db_val (nano::uint256_union const & val_a) : - db_val (sizeof (val_a), const_cast (&val_a)) - { - } - - db_val (nano::uint512_union const & val_a) : - db_val (sizeof (val_a), const_cast (&val_a)) - { - } - - db_val (nano::qualified_root const & val_a) : - db_val (sizeof (val_a), const_cast (&val_a)) - { - } - - db_val (nano::account_info const & val_a) : - db_val (val_a.db_size (), const_cast (&val_a)) - { - } - - db_val (nano::account_info_v14 const & val_a) : - db_val (val_a.db_size (), const_cast (&val_a)) - { - } - - db_val (nano::pending_info const & val_a) : - db_val (val_a.db_size (), const_cast (&val_a)) - { - static_assert (std::is_standard_layout::value, "Standard layout is required"); - } - - db_val (nano::pending_info_v14 const & val_a) : - db_val (val_a.db_size (), const_cast (&val_a)) - { - static_assert (std::is_standard_layout::value, "Standard layout is required"); - } - - db_val (nano::pending_key const & val_a) : - db_val (sizeof (val_a), const_cast (&val_a)) - { - static_assert (std::is_standard_layout::value, "Standard layout is required"); - } - - db_val (nano::confirmation_height_info const & val_a) : - buffer (std::make_shared> ()) - { - { - nano::vectorstream stream (*buffer); - val_a.serialize (stream); - } - convert_buffer_to_value (); - } - - db_val (nano::block_info const & val_a) : - db_val (sizeof (val_a), const_cast (&val_a)) - { - static_assert (std::is_standard_layout::value, "Standard layout is required"); - } - - db_val (nano::endpoint_key const & val_a) : - db_val (sizeof (val_a), const_cast (&val_a)) - { - static_assert (std::is_standard_layout::value, "Standard layout is required"); - } - - db_val (std::shared_ptr const & val_a) : - buffer (std::make_shared> ()) - { - { - nano::vectorstream stream (*buffer); - nano::serialize_block (stream, *val_a); - } - convert_buffer_to_value (); - } - - db_val (uint64_t val_a) : - buffer (std::make_shared> ()) - { - { - boost::endian::native_to_big_inplace (val_a); - nano::vectorstream stream (*buffer); - nano::write (stream, val_a); - } - convert_buffer_to_value (); - } - - explicit operator nano::account_info () const - { - nano::account_info result; - debug_assert (size () == result.db_size ()); - std::copy (reinterpret_cast (data ()), reinterpret_cast (data ()) + result.db_size (), reinterpret_cast (&result)); - return result; - } - - explicit operator nano::account_info_v14 () const - { - nano::account_info_v14 result; - debug_assert (size () == result.db_size ()); - std::copy (reinterpret_cast (data ()), reinterpret_cast (data ()) + result.db_size (), reinterpret_cast (&result)); - return result; - } - - explicit operator nano::block_info () const - { - nano::block_info result; - debug_assert (size () == sizeof (result)); - static_assert (sizeof (nano::block_info::account) + sizeof (nano::block_info::balance) == sizeof (result), "Packed class"); - std::copy (reinterpret_cast (data ()), reinterpret_cast (data ()) + sizeof (result), reinterpret_cast (&result)); - return result; - } - - explicit operator nano::pending_info_v14 () const - { - nano::pending_info_v14 result; - debug_assert (size () == result.db_size ()); - std::copy (reinterpret_cast (data ()), reinterpret_cast (data ()) + result.db_size (), reinterpret_cast (&result)); - return result; - } - - explicit operator nano::pending_info () const - { - nano::pending_info result; - debug_assert (size () == result.db_size ()); - std::copy (reinterpret_cast (data ()), reinterpret_cast (data ()) + result.db_size (), reinterpret_cast (&result)); - return result; - } - - explicit operator nano::pending_key () const - { - nano::pending_key result; - debug_assert (size () == sizeof (result)); - static_assert (sizeof (nano::pending_key::account) + sizeof (nano::pending_key::hash) == sizeof (result), "Packed class"); - std::copy (reinterpret_cast (data ()), reinterpret_cast (data ()) + sizeof (result), reinterpret_cast (&result)); - return result; - } - - explicit operator nano::confirmation_height_info () const - { - nano::bufferstream stream (reinterpret_cast (data ()), size ()); - nano::confirmation_height_info result; - bool error (result.deserialize (stream)); - (void)error; - debug_assert (!error); - return result; - } - - explicit operator nano::uint128_union () const - { - return convert (); - } - - explicit operator nano::amount () const - { - return convert (); - } - - explicit operator nano::block_hash () const - { - return convert (); - } - - explicit operator nano::public_key () const - { - return convert (); - } - - explicit operator nano::qualified_root () const - { - return convert (); - } - - explicit operator nano::uint256_union () const - { - return convert (); - } - - explicit operator nano::uint512_union () const - { - return convert (); - } - - explicit operator std::array () const - { - nano::bufferstream stream (reinterpret_cast (data ()), size ()); - std::array result; - auto error = nano::try_read (stream, result); - (void)error; - debug_assert (!error); - return result; - } - - explicit operator nano::endpoint_key () const - { - nano::endpoint_key result; - std::copy (reinterpret_cast (data ()), reinterpret_cast (data ()) + sizeof (result), reinterpret_cast (&result)); - return result; - } - - template - explicit operator block_w_sideband_v18 () const - { - nano::bufferstream stream (reinterpret_cast (data ()), size ()); - auto error (false); - block_w_sideband_v18 block_w_sideband; - block_w_sideband.block = std::make_shared (error, stream); - release_assert (!error); - - error = block_w_sideband.sideband.deserialize (stream, block_w_sideband.block->type ()); - release_assert (!error); - - return block_w_sideband; - } - - explicit operator block_w_sideband () const - { - nano::bufferstream stream (reinterpret_cast (data ()), size ()); - nano::block_w_sideband block_w_sideband; - block_w_sideband.block = (nano::deserialize_block (stream)); - auto error = block_w_sideband.sideband.deserialize (stream, block_w_sideband.block->type ()); - release_assert (!error); - block_w_sideband.block->sideband_set (block_w_sideband.sideband); - return block_w_sideband; - } - - explicit operator state_block_w_sideband_v14 () const - { - nano::bufferstream stream (reinterpret_cast (data ()), size ()); - auto error (false); - nano::state_block_w_sideband_v14 block_w_sideband; - block_w_sideband.state_block = std::make_shared (error, stream); - debug_assert (!error); - - block_w_sideband.sideband.type = nano::block_type::state; - error = block_w_sideband.sideband.deserialize (stream); - debug_assert (!error); - - return block_w_sideband; - } - - explicit operator std::nullptr_t () const - { - return nullptr; - } - - explicit operator nano::no_value () const - { - return no_value::dummy; - } - - explicit operator std::shared_ptr () const - { - nano::bufferstream stream (reinterpret_cast (data ()), size ()); - std::shared_ptr result (nano::deserialize_block (stream)); - return result; - } - - template - std::shared_ptr convert_to_block () const - { - nano::bufferstream stream (reinterpret_cast (data ()), size ()); - auto error (false); - auto result (std::make_shared (error, stream)); - debug_assert (!error); - return result; - } - - explicit operator std::shared_ptr () const - { - return convert_to_block (); - } - - explicit operator std::shared_ptr () const - { - return convert_to_block (); - } - - explicit operator std::shared_ptr () const - { - return convert_to_block (); - } - - explicit operator std::shared_ptr () const - { - return convert_to_block (); - } - - explicit operator std::shared_ptr () const - { - return convert_to_block (); - } - - explicit operator std::shared_ptr () const - { - nano::bufferstream stream (reinterpret_cast (data ()), size ()); - auto error (false); - auto result (nano::make_shared (error, stream)); - debug_assert (!error); - return result; - } - - explicit operator uint64_t () const - { - uint64_t result; - nano::bufferstream stream (reinterpret_cast (data ()), size ()); - auto error (nano::try_read (stream, result)); - (void)error; - debug_assert (!error); - boost::endian::big_to_native_inplace (result); - return result; - } - - operator Val * () const - { - // Allow passing a temporary to a non-c++ function which doesn't have constness - return const_cast (&value); - } - - operator Val const & () const - { - return value; - } - - // Must be specialized - void * data () const; - size_t size () const; - db_val (size_t size_a, void * data_a); - void convert_buffer_to_value (); - - Val value; - std::shared_ptr> buffer; - -private: - template - T convert () const - { - T result; - debug_assert (size () == sizeof (result)); - std::copy (reinterpret_cast (data ()), reinterpret_cast (data ()) + sizeof (result), result.bytes.data ()); - return result; - } -}; - -class transaction; -class store; - -/** - * Determine the representative for this block - */ -class representative_visitor final : public nano::block_visitor -{ -public: - representative_visitor (nano::transaction const & transaction_a, nano::store & store_a); - ~representative_visitor () = default; - void compute (nano::block_hash const & hash_a); - void send_block (nano::send_block const & block_a) override; - void receive_block (nano::receive_block const & block_a) override; - void open_block (nano::open_block const & block_a) override; - void change_block (nano::change_block const & block_a) override; - void state_block (nano::state_block const & block_a) override; - nano::transaction const & transaction; - nano::store & store; - nano::block_hash current; - nano::block_hash result; -}; -template -class store_iterator_impl -{ -public: - virtual ~store_iterator_impl () = default; - virtual nano::store_iterator_impl & operator++ () = 0; - virtual nano::store_iterator_impl & operator-- () = 0; - virtual bool operator== (nano::store_iterator_impl const & other_a) const = 0; - virtual bool is_end_sentinal () const = 0; - virtual void fill (std::pair &) const = 0; - nano::store_iterator_impl & operator= (nano::store_iterator_impl const &) = delete; - bool operator== (nano::store_iterator_impl const * other_a) const - { - return (other_a != nullptr && *this == *other_a) || (other_a == nullptr && is_end_sentinal ()); - } - bool operator!= (nano::store_iterator_impl const & other_a) const - { - return !(*this == other_a); - } -}; -/** - * Iterates the key/value pairs of a transaction - */ -template -class store_iterator final -{ -public: - store_iterator (std::nullptr_t) - { - } - store_iterator (std::unique_ptr> impl_a) : - impl (std::move (impl_a)) - { - impl->fill (current); - } - store_iterator (nano::store_iterator && other_a) : - current (std::move (other_a.current)), - impl (std::move (other_a.impl)) - { - } - nano::store_iterator & operator++ () - { - ++*impl; - impl->fill (current); - return *this; - } - nano::store_iterator & operator-- () - { - --*impl; - impl->fill (current); - return *this; - } - nano::store_iterator & operator= (nano::store_iterator && other_a) noexcept - { - impl = std::move (other_a.impl); - current = std::move (other_a.current); - return *this; - } - nano::store_iterator & operator= (nano::store_iterator const &) = delete; - std::pair * operator-> () - { - return ¤t; - } - bool operator== (nano::store_iterator const & other_a) const - { - return (impl == nullptr && other_a.impl == nullptr) || (impl != nullptr && *impl == other_a.impl.get ()) || (other_a.impl != nullptr && *other_a.impl == impl.get ()); - } - bool operator!= (nano::store_iterator const & other_a) const - { - return !(*this == other_a); - } - -private: - std::pair current; - std::unique_ptr> impl; -}; - -// Keep this in alphabetical order -enum class tables -{ - accounts, - blocks, - confirmation_height, - default_unused, // RocksDB only - final_votes, - frontiers, - meta, - online_weight, - peers, - pending, - pruned, - vote -}; - -class transaction_impl -{ -public: - virtual ~transaction_impl () = default; - virtual void * get_handle () const = 0; -}; - -class read_transaction_impl : public transaction_impl -{ -public: - virtual void reset () = 0; - virtual void renew () = 0; -}; - -class write_transaction_impl : public transaction_impl -{ -public: - virtual void commit () = 0; - virtual void renew () = 0; - virtual bool contains (nano::tables table_a) const = 0; -}; - -class transaction -{ -public: - virtual ~transaction () = default; - virtual void * get_handle () const = 0; -}; - -/** - * RAII wrapper of a read MDB_txn where the constructor starts the transaction - * and the destructor aborts it. - */ -class read_transaction final : public transaction -{ -public: - explicit read_transaction (std::unique_ptr read_transaction_impl); - void * get_handle () const override; - void reset () const; - void renew () const; - void refresh () const; - -private: - std::unique_ptr impl; -}; - -/** - * RAII wrapper of a read-write MDB_txn where the constructor starts the transaction - * and the destructor commits it. - */ -class write_transaction final : public transaction -{ -public: - explicit write_transaction (std::unique_ptr write_transaction_impl); - void * get_handle () const override; - void commit (); - void renew (); - void refresh (); - bool contains (nano::tables table_a) const; - -private: - std::unique_ptr impl; -}; - -class ledger_cache; - -/** - * Manages frontier storage and iteration - */ -class frontier_store -{ -public: - virtual void put (nano::write_transaction const &, nano::block_hash const &, nano::account const &) = 0; - virtual nano::account get (nano::transaction const &, nano::block_hash const &) const = 0; - virtual void del (nano::write_transaction const &, nano::block_hash const &) = 0; - virtual nano::store_iterator begin (nano::transaction const &) const = 0; - virtual nano::store_iterator begin (nano::transaction const &, nano::block_hash const &) const = 0; - virtual nano::store_iterator end () const = 0; - virtual void for_each_par (std::function, nano::store_iterator)> const & action_a) const = 0; -}; - -/** - * Manages account storage and iteration - */ -class account_store -{ -public: - virtual void put (nano::write_transaction const &, nano::account const &, nano::account_info const &) = 0; - virtual bool get (nano::transaction const &, nano::account const &, nano::account_info &) = 0; - std::optional get (nano::transaction const &, nano::account const &); - virtual void del (nano::write_transaction const &, nano::account const &) = 0; - virtual bool exists (nano::transaction const &, nano::account const &) = 0; - virtual size_t count (nano::transaction const &) = 0; - virtual nano::store_iterator begin (nano::transaction const &, nano::account const &) const = 0; - virtual nano::store_iterator begin (nano::transaction const &) const = 0; - virtual nano::store_iterator rbegin (nano::transaction const &) const = 0; - virtual nano::store_iterator end () const = 0; - virtual void for_each_par (std::function, nano::store_iterator)> const &) const = 0; -}; - -/** - * Manages pending storage and iteration - */ -class pending_store -{ -public: - virtual void put (nano::write_transaction const &, nano::pending_key const &, nano::pending_info const &) = 0; - virtual void del (nano::write_transaction const &, nano::pending_key const &) = 0; - virtual bool get (nano::transaction const &, nano::pending_key const &, nano::pending_info &) = 0; - virtual bool exists (nano::transaction const &, nano::pending_key const &) = 0; - virtual bool any (nano::transaction const &, nano::account const &) = 0; - virtual nano::store_iterator begin (nano::transaction const &, nano::pending_key const &) const = 0; - virtual nano::store_iterator begin (nano::transaction const &) const = 0; - virtual nano::store_iterator end () const = 0; - virtual void for_each_par (std::function, nano::store_iterator)> const & action_a) const = 0; -}; - -/** - * Manages peer storage and iteration - */ -class peer_store -{ -public: - virtual void put (nano::write_transaction const & transaction_a, nano::endpoint_key const & endpoint_a) = 0; - virtual void del (nano::write_transaction const & transaction_a, nano::endpoint_key const & endpoint_a) = 0; - virtual bool exists (nano::transaction const & transaction_a, nano::endpoint_key const & endpoint_a) const = 0; - virtual size_t count (nano::transaction const & transaction_a) const = 0; - virtual void clear (nano::write_transaction const & transaction_a) = 0; - virtual nano::store_iterator begin (nano::transaction const & transaction_a) const = 0; - virtual nano::store_iterator end () const = 0; -}; - -/** - * Manages online weight storage and iteration - */ -class online_weight_store -{ -public: - virtual void put (nano::write_transaction const &, uint64_t, nano::amount const &) = 0; - virtual void del (nano::write_transaction const &, uint64_t) = 0; - virtual nano::store_iterator begin (nano::transaction const &) const = 0; - virtual nano::store_iterator rbegin (nano::transaction const &) const = 0; - virtual nano::store_iterator end () const = 0; - virtual size_t count (nano::transaction const &) const = 0; - virtual void clear (nano::write_transaction const &) = 0; -}; - -/** - * Manages pruned storage and iteration - */ -class pruned_store -{ -public: - virtual void put (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a) = 0; - virtual void del (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a) = 0; - virtual bool exists (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const = 0; - virtual nano::block_hash random (nano::transaction const & transaction_a) = 0; - virtual size_t count (nano::transaction const & transaction_a) const = 0; - virtual void clear (nano::write_transaction const &) = 0; - virtual nano::store_iterator begin (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const = 0; - virtual nano::store_iterator begin (nano::transaction const & transaction_a) const = 0; - virtual nano::store_iterator end () const = 0; - virtual void for_each_par (std::function, nano::store_iterator)> const & action_a) const = 0; -}; - -/** - * Manages confirmation height storage and iteration - */ -class confirmation_height_store -{ -public: - virtual void put (nano::write_transaction const & transaction_a, nano::account const & account_a, nano::confirmation_height_info const & confirmation_height_info_a) = 0; - - /** Retrieves confirmation height info relating to an account. - * The parameter confirmation_height_info_a is always written. - * On error, the confirmation height and frontier hash are set to 0. - * Ruturns true on error, false on success. - */ - virtual bool get (nano::transaction const & transaction_a, nano::account const & account_a, nano::confirmation_height_info & confirmation_height_info_a) = 0; - std::optional get (nano::transaction const & transaction_a, nano::account const & account_a); - virtual bool exists (nano::transaction const & transaction_a, nano::account const & account_a) const = 0; - virtual void del (nano::write_transaction const & transaction_a, nano::account const & account_a) = 0; - virtual uint64_t count (nano::transaction const & transaction_a) = 0; - virtual void clear (nano::write_transaction const &, nano::account const &) = 0; - virtual void clear (nano::write_transaction const &) = 0; - virtual nano::store_iterator begin (nano::transaction const & transaction_a, nano::account const & account_a) const = 0; - virtual nano::store_iterator begin (nano::transaction const & transaction_a) const = 0; - virtual nano::store_iterator end () const = 0; - virtual void for_each_par (std::function, nano::store_iterator)> const &) const = 0; -}; - -/** - * Manages final vote storage and iteration - */ -class final_vote_store -{ -public: - virtual bool put (nano::write_transaction const & transaction_a, nano::qualified_root const & root_a, nano::block_hash const & hash_a) = 0; - virtual std::vector get (nano::transaction const & transaction_a, nano::root const & root_a) = 0; - virtual void del (nano::write_transaction const & transaction_a, nano::root const & root_a) = 0; - virtual size_t count (nano::transaction const & transaction_a) const = 0; - virtual void clear (nano::write_transaction const &, nano::root const &) = 0; - virtual void clear (nano::write_transaction const &) = 0; - virtual nano::store_iterator begin (nano::transaction const & transaction_a, nano::qualified_root const & root_a) const = 0; - virtual nano::store_iterator begin (nano::transaction const & transaction_a) const = 0; - virtual nano::store_iterator end () const = 0; - virtual void for_each_par (std::function, nano::store_iterator)> const & action_a) const = 0; -}; - -/** - * Manages version storage - */ -class version_store -{ -public: - virtual void put (nano::write_transaction const &, int) = 0; - virtual int get (nano::transaction const &) const = 0; -}; - -/** - * Manages block storage and iteration - */ -class block_store -{ -public: - virtual void put (nano::write_transaction const &, nano::block_hash const &, nano::block const &) = 0; - virtual void raw_put (nano::write_transaction const &, std::vector const &, nano::block_hash const &) = 0; - virtual nano::block_hash successor (nano::transaction const &, nano::block_hash const &) const = 0; - virtual void successor_clear (nano::write_transaction const &, nano::block_hash const &) = 0; - virtual std::shared_ptr get (nano::transaction const &, nano::block_hash const &) const = 0; - virtual std::shared_ptr get_no_sideband (nano::transaction const &, nano::block_hash const &) const = 0; - virtual std::shared_ptr random (nano::transaction const &) = 0; - virtual void del (nano::write_transaction const &, nano::block_hash const &) = 0; - virtual bool exists (nano::transaction const &, nano::block_hash const &) = 0; - virtual uint64_t count (nano::transaction const &) = 0; - virtual nano::account account (nano::transaction const &, nano::block_hash const &) const = 0; - virtual nano::account account_calculated (nano::block const &) const = 0; - virtual nano::store_iterator begin (nano::transaction const &, nano::block_hash const &) const = 0; - virtual nano::store_iterator begin (nano::transaction const &) const = 0; - virtual nano::store_iterator end () const = 0; - virtual nano::uint128_t balance (nano::transaction const &, nano::block_hash const &) = 0; - virtual nano::uint128_t balance_calculated (std::shared_ptr const &) const = 0; - virtual nano::epoch version (nano::transaction const &, nano::block_hash const &) = 0; - virtual void for_each_par (std::function, nano::store_iterator)> const & action_a) const = 0; - virtual uint64_t account_height (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const = 0; -}; - -/** - * Store manager - */ -class store -{ - friend class rocksdb_block_store_tombstone_count_Test; - friend class mdb_block_store_upgrade_v21_v22_Test; - -public: - // clang-format off - explicit store ( - nano::block_store &, - nano::frontier_store &, - nano::account_store &, - nano::pending_store &, - nano::online_weight_store &, - nano::pruned_store &, - nano::peer_store &, - nano::confirmation_height_store &, - nano::final_vote_store &, - nano::version_store & - ); - // clang-format on - virtual ~store () = default; - void initialize (nano::write_transaction const & transaction_a, nano::ledger_cache & ledger_cache_a, nano::ledger_constants & constants); - virtual uint64_t count (nano::transaction const & transaction_a, tables table_a) const = 0; - virtual int drop (nano::write_transaction const & transaction_a, tables table_a) = 0; - virtual bool not_found (int status) const = 0; - virtual bool success (int status) const = 0; - virtual int status_code_not_found () const = 0; - virtual std::string error_string (int status) const = 0; - - block_store & block; - frontier_store & frontier; - account_store & account; - pending_store & pending; - static int constexpr version_minimum{ 14 }; - static int constexpr version_current{ 22 }; - -public: - online_weight_store & online_weight; - pruned_store & pruned; - peer_store & peer; - confirmation_height_store & confirmation_height; - final_vote_store & final_vote; - version_store & version; - - virtual unsigned max_block_write_batch_num () const = 0; - - virtual bool copy_db (boost::filesystem::path const & destination) = 0; - virtual void rebuild_db (nano::write_transaction const & transaction_a) = 0; - - /** Not applicable to all sub-classes */ - virtual void serialize_mdb_tracker (boost::property_tree::ptree &, std::chrono::milliseconds, std::chrono::milliseconds){}; - virtual void serialize_memory_stats (boost::property_tree::ptree &) = 0; - - virtual bool init_error () const = 0; - - /** Start read-write transaction */ - virtual nano::write_transaction tx_begin_write (std::vector const & tables_to_lock = {}, std::vector const & tables_no_lock = {}) = 0; - - /** Start read-only transaction */ - virtual nano::read_transaction tx_begin_read () const = 0; - - virtual std::string vendor_get () const = 0; -}; - -std::unique_ptr make_store (nano::logger_mt & logger, boost::filesystem::path const & path, nano::ledger_constants & constants, bool open_read_only = false, bool add_db_postfix = true, nano::rocksdb_config const & rocksdb_config = nano::rocksdb_config{}, nano::txn_tracking_config const & txn_tracking_config_a = nano::txn_tracking_config{}, std::chrono::milliseconds block_processor_batch_max_time_a = std::chrono::milliseconds (5000), nano::lmdb_config const & lmdb_config_a = nano::lmdb_config{}, bool backup_before_upgrade = false); -} - -namespace std -{ -template <> -struct hash<::nano::tables> -{ - size_t operator() (::nano::tables const & table_a) const - { - return static_cast (table_a); - } -}; -} diff --git a/nano/secure/versioning.cpp b/nano/secure/versioning.cpp deleted file mode 100644 index 1ab31d5819..0000000000 --- a/nano/secure/versioning.cpp +++ /dev/null @@ -1,249 +0,0 @@ -#include - -#include - -#include - -nano::pending_info_v14::pending_info_v14 (nano::account const & source_a, nano::amount const & amount_a, nano::epoch epoch_a) : - source (source_a), - amount (amount_a), - epoch (epoch_a) -{ -} - -bool nano::pending_info_v14::deserialize (nano::stream & stream_a) -{ - auto error (false); - try - { - nano::read (stream_a, source.bytes); - nano::read (stream_a, amount.bytes); - } - catch (std::runtime_error const &) - { - error = true; - } - - return error; -} - -size_t nano::pending_info_v14::db_size () const -{ - return sizeof (source) + sizeof (amount); -} - -bool nano::pending_info_v14::operator== (nano::pending_info_v14 const & other_a) const -{ - return source == other_a.source && amount == other_a.amount && epoch == other_a.epoch; -} - -nano::account_info_v14::account_info_v14 (nano::block_hash const & head_a, nano::block_hash const & rep_block_a, nano::block_hash const & open_block_a, nano::amount const & balance_a, uint64_t modified_a, uint64_t block_count_a, uint64_t confirmation_height_a, nano::epoch epoch_a) : - head (head_a), - rep_block (rep_block_a), - open_block (open_block_a), - balance (balance_a), - modified (modified_a), - block_count (block_count_a), - confirmation_height (confirmation_height_a), - epoch (epoch_a) -{ -} - -size_t nano::account_info_v14::db_size () const -{ - debug_assert (reinterpret_cast (this) == reinterpret_cast (&head)); - debug_assert (reinterpret_cast (&head) + sizeof (head) == reinterpret_cast (&rep_block)); - debug_assert (reinterpret_cast (&rep_block) + sizeof (rep_block) == reinterpret_cast (&open_block)); - debug_assert (reinterpret_cast (&open_block) + sizeof (open_block) == reinterpret_cast (&balance)); - debug_assert (reinterpret_cast (&balance) + sizeof (balance) == reinterpret_cast (&modified)); - debug_assert (reinterpret_cast (&modified) + sizeof (modified) == reinterpret_cast (&block_count)); - debug_assert (reinterpret_cast (&block_count) + sizeof (block_count) == reinterpret_cast (&confirmation_height)); - return sizeof (head) + sizeof (rep_block) + sizeof (open_block) + sizeof (balance) + sizeof (modified) + sizeof (block_count) + sizeof (confirmation_height); -} - -nano::block_sideband_v14::block_sideband_v14 (nano::block_type type_a, nano::account const & account_a, nano::block_hash const & successor_a, nano::amount const & balance_a, uint64_t height_a, uint64_t timestamp_a) : - type (type_a), - successor (successor_a), - account (account_a), - balance (balance_a), - height (height_a), - timestamp (timestamp_a) -{ -} - -size_t nano::block_sideband_v14::size (nano::block_type type_a) -{ - size_t result (0); - result += sizeof (successor); - if (type_a != nano::block_type::state && type_a != nano::block_type::open) - { - result += sizeof (account); - } - if (type_a != nano::block_type::open) - { - result += sizeof (height); - } - if (type_a == nano::block_type::receive || type_a == nano::block_type::change || type_a == nano::block_type::open) - { - result += sizeof (balance); - } - result += sizeof (timestamp); - return result; -} - -void nano::block_sideband_v14::serialize (nano::stream & stream_a) const -{ - nano::write (stream_a, successor.bytes); - if (type != nano::block_type::state && type != nano::block_type::open) - { - nano::write (stream_a, account.bytes); - } - if (type != nano::block_type::open) - { - nano::write (stream_a, boost::endian::native_to_big (height)); - } - if (type == nano::block_type::receive || type == nano::block_type::change || type == nano::block_type::open) - { - nano::write (stream_a, balance.bytes); - } - nano::write (stream_a, boost::endian::native_to_big (timestamp)); -} - -bool nano::block_sideband_v14::deserialize (nano::stream & stream_a) -{ - bool result (false); - try - { - nano::read (stream_a, successor.bytes); - if (type != nano::block_type::state && type != nano::block_type::open) - { - nano::read (stream_a, account.bytes); - } - if (type != nano::block_type::open) - { - nano::read (stream_a, height); - boost::endian::big_to_native_inplace (height); - } - else - { - height = 1; - } - if (type == nano::block_type::receive || type == nano::block_type::change || type == nano::block_type::open) - { - nano::read (stream_a, balance.bytes); - } - nano::read (stream_a, timestamp); - boost::endian::big_to_native_inplace (timestamp); - } - catch (std::runtime_error &) - { - result = true; - } - - return result; -} - -nano::block_sideband_v18::block_sideband_v18 (nano::account const & account_a, nano::block_hash const & successor_a, nano::amount const & balance_a, uint64_t height_a, uint64_t timestamp_a, nano::block_details const & details_a) : - successor (successor_a), - account (account_a), - balance (balance_a), - height (height_a), - timestamp (timestamp_a), - details (details_a) -{ -} - -nano::block_sideband_v18::block_sideband_v18 (nano::account const & account_a, nano::block_hash const & successor_a, nano::amount const & balance_a, uint64_t height_a, uint64_t timestamp_a, nano::epoch epoch_a, bool is_send, bool is_receive, bool is_epoch) : - successor (successor_a), - account (account_a), - balance (balance_a), - height (height_a), - timestamp (timestamp_a), - details (epoch_a, is_send, is_receive, is_epoch) -{ -} - -size_t nano::block_sideband_v18::size (nano::block_type type_a) -{ - size_t result (0); - result += sizeof (successor); - if (type_a != nano::block_type::state && type_a != nano::block_type::open) - { - result += sizeof (account); - } - if (type_a != nano::block_type::open) - { - result += sizeof (height); - } - if (type_a == nano::block_type::receive || type_a == nano::block_type::change || type_a == nano::block_type::open) - { - result += sizeof (balance); - } - result += sizeof (timestamp); - if (type_a == nano::block_type::state) - { - static_assert (sizeof (nano::epoch) == nano::block_details::size (), "block_details_v18 is larger than the epoch enum"); - result += nano::block_details::size (); - } - return result; -} - -void nano::block_sideband_v18::serialize (nano::stream & stream_a, nano::block_type type_a) const -{ - nano::write (stream_a, successor.bytes); - if (type_a != nano::block_type::state && type_a != nano::block_type::open) - { - nano::write (stream_a, account.bytes); - } - if (type_a != nano::block_type::open) - { - nano::write (stream_a, boost::endian::native_to_big (height)); - } - if (type_a == nano::block_type::receive || type_a == nano::block_type::change || type_a == nano::block_type::open) - { - nano::write (stream_a, balance.bytes); - } - nano::write (stream_a, boost::endian::native_to_big (timestamp)); - if (type_a == nano::block_type::state) - { - details.serialize (stream_a); - } -} - -bool nano::block_sideband_v18::deserialize (nano::stream & stream_a, nano::block_type type_a) -{ - bool result (false); - try - { - nano::read (stream_a, successor.bytes); - if (type_a != nano::block_type::state && type_a != nano::block_type::open) - { - nano::read (stream_a, account.bytes); - } - if (type_a != nano::block_type::open) - { - nano::read (stream_a, height); - boost::endian::big_to_native_inplace (height); - } - else - { - height = 1; - } - if (type_a == nano::block_type::receive || type_a == nano::block_type::change || type_a == nano::block_type::open) - { - nano::read (stream_a, balance.bytes); - } - nano::read (stream_a, timestamp); - boost::endian::big_to_native_inplace (timestamp); - if (type_a == nano::block_type::state) - { - result = details.deserialize (stream_a); - } - } - catch (std::runtime_error &) - { - result = true; - } - - return result; -} diff --git a/nano/secure/versioning.hpp b/nano/secure/versioning.hpp deleted file mode 100644 index 429de22d97..0000000000 --- a/nano/secure/versioning.hpp +++ /dev/null @@ -1,74 +0,0 @@ -#pragma once - -#include -#include - -struct MDB_val; - -namespace nano -{ -class pending_info_v14 final -{ -public: - pending_info_v14 () = default; - pending_info_v14 (nano::account const &, nano::amount const &, nano::epoch); - size_t db_size () const; - bool deserialize (nano::stream &); - bool operator== (nano::pending_info_v14 const &) const; - nano::account source{}; - nano::amount amount{ 0 }; - nano::epoch epoch{ nano::epoch::epoch_0 }; -}; -class account_info_v14 final -{ -public: - account_info_v14 () = default; - account_info_v14 (nano::block_hash const &, nano::block_hash const &, nano::block_hash const &, nano::amount const &, uint64_t, uint64_t, uint64_t, nano::epoch); - size_t db_size () const; - nano::block_hash head{ 0 }; - nano::block_hash rep_block{ 0 }; - nano::block_hash open_block{ 0 }; - nano::amount balance{ 0 }; - uint64_t modified{ 0 }; - uint64_t block_count{ 0 }; - uint64_t confirmation_height{ 0 }; - nano::epoch epoch{ nano::epoch::epoch_0 }; -}; -class block_sideband_v14 final -{ -public: - block_sideband_v14 () = default; - block_sideband_v14 (nano::block_type, nano::account const &, nano::block_hash const &, nano::amount const &, uint64_t, uint64_t); - void serialize (nano::stream &) const; - bool deserialize (nano::stream &); - static size_t size (nano::block_type); - nano::block_type type{ nano::block_type::invalid }; - nano::block_hash successor{ 0 }; - nano::account account{}; - nano::amount balance{ 0 }; - uint64_t height{ 0 }; - uint64_t timestamp{ 0 }; -}; -class state_block_w_sideband_v14 -{ -public: - std::shared_ptr state_block; - nano::block_sideband_v14 sideband; -}; -class block_sideband_v18 final -{ -public: - block_sideband_v18 () = default; - block_sideband_v18 (nano::account const &, nano::block_hash const &, nano::amount const &, uint64_t, uint64_t, nano::block_details const &); - block_sideband_v18 (nano::account const &, nano::block_hash const &, nano::amount const &, uint64_t, uint64_t, nano::epoch, bool is_send, bool is_receive, bool is_epoch); - void serialize (nano::stream &, nano::block_type) const; - bool deserialize (nano::stream &, nano::block_type); - static size_t size (nano::block_type); - nano::block_hash successor{ 0 }; - nano::account account{}; - nano::amount balance{ 0 }; - uint64_t height{ 0 }; - uint64_t timestamp{ 0 }; - nano::block_details details; -}; -} diff --git a/nano/slow_test/bootstrap.cpp b/nano/slow_test/bootstrap.cpp index 7004a74cad..b722cf73e1 100644 --- a/nano/slow_test/bootstrap.cpp +++ b/nano/slow_test/bootstrap.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include diff --git a/nano/slow_test/node.cpp b/nano/slow_test/node.cpp index d63b2ac154..740383a83c 100644 --- a/nano/slow_test/node.cpp +++ b/nano/slow_test/node.cpp @@ -1,6 +1,8 @@ #include +#include #include #include +#include #include #include #include @@ -24,7 +26,7 @@ using namespace std::chrono_literals; * function to count the block in the pruned store one by one * we manually count the blocks one by one because the rocksdb count feature is not accurate */ -size_t manually_count_pruned_blocks (nano::store & store) +size_t manually_count_pruned_blocks (nano::store::component & store) { size_t count = 0; auto transaction = store.tx_begin_read (); @@ -1015,7 +1017,7 @@ TEST (confirmation_height, dynamic_algorithm_no_transition_while_pending) std::vector> state_blocks; auto const num_blocks = nano::confirmation_height::unbounded_cutoff - 2; - auto add_block_to_genesis_chain = [&] (nano::write_transaction & transaction) { + auto add_block_to_genesis_chain = [&] (store::write_transaction & transaction) { static int num = 0; nano::block_builder builder; auto send = builder diff --git a/nano/store/CMakeLists.txt b/nano/store/CMakeLists.txt new file mode 100644 index 0000000000..f494624998 --- /dev/null +++ b/nano/store/CMakeLists.txt @@ -0,0 +1,109 @@ +add_library( + nano_store + account.hpp + block.hpp + component.hpp + confirmation_height.hpp + db_val.hpp + iterator.hpp + iterator_impl.hpp + final.hpp + frontier.hpp + lmdb/account.hpp + lmdb/block.hpp + lmdb/confirmation_height.hpp + lmdb/db_val.hpp + lmdb/final_vote.hpp + lmdb/frontier.hpp + lmdb/iterator.hpp + lmdb/lmdb.hpp + lmdb/lmdb_env.hpp + lmdb/iterator.hpp + lmdb/transaction_impl.hpp + lmdb/online_weight.hpp + lmdb/peer.hpp + lmdb/pending.hpp + lmdb/pruned.hpp + lmdb/transaction_impl.hpp + lmdb/version.hpp + lmdb/wallet_value.hpp + online_weight.hpp + peer.hpp + pending.hpp + pruned.hpp + rocksdb/account.hpp + rocksdb/block.hpp + rocksdb/confirmation_height.hpp + rocksdb/db_val.hpp + rocksdb/final_vote.hpp + rocksdb/frontier.hpp + rocksdb/iterator.hpp + rocksdb/online_weight.hpp + rocksdb/peer.hpp + rocksdb/pending.hpp + rocksdb/pruned.hpp + rocksdb/rocksdb.hpp + rocksdb/iterator.hpp + rocksdb/transaction_impl.hpp + rocksdb/version.hpp + tables.hpp + transaction.hpp + version.hpp + versioning.hpp + account.cpp + block.cpp + component.cpp + confirmation_height.cpp + db_val.cpp + iterator.cpp + iterator_impl.cpp + final.cpp + frontier.cpp + lmdb/account.cpp + lmdb/block.cpp + lmdb/confirmation_height.cpp + lmdb/db_val.cpp + lmdb/final_vote.cpp + lmdb/frontier.cpp + lmdb/lmdb.cpp + lmdb/lmdb_env.cpp + lmdb/transaction.cpp + lmdb/online_weight.cpp + lmdb/peer.cpp + lmdb/pending.cpp + lmdb/pruned.cpp + lmdb/version.cpp + lmdb/wallet_value.cpp + online_weight.cpp + peer.cpp + pending.cpp + pruned.cpp + rocksdb/account.cpp + rocksdb/block.cpp + rocksdb/confirmation_height.cpp + rocksdb/db_val.cpp + rocksdb/final_vote.cpp + rocksdb/frontier.cpp + rocksdb/online_weight.cpp + rocksdb/peer.cpp + rocksdb/pending.cpp + rocksdb/pruned.cpp + rocksdb/rocksdb.cpp + rocksdb/transaction.cpp + rocksdb/version.cpp + transaction.cpp + version.cpp + versioning.cpp) + +target_link_libraries( + nano_store + Boost::circular_buffer + Boost::endian + Boost::filesystem + Boost::iostreams + Boost::log_setup + Boost::log + Boost::multiprecision + Boost::pool + Boost::stacktrace + Boost::variant) diff --git a/nano/store/account.cpp b/nano/store/account.cpp new file mode 100644 index 0000000000..a3bb7a1970 --- /dev/null +++ b/nano/store/account.cpp @@ -0,0 +1,15 @@ +#include + +std::optional nano::store::account::get (store::transaction const & transaction, nano::account const & account) +{ + nano::account_info info; + bool error = get (transaction, account, info); + if (!error) + { + return info; + } + else + { + return std::nullopt; + } +} diff --git a/nano/store/account.hpp b/nano/store/account.hpp new file mode 100644 index 0000000000..9d20fdd384 --- /dev/null +++ b/nano/store/account.hpp @@ -0,0 +1,33 @@ +#pragma once + +#include +#include +#include + +#include + +namespace nano +{ +class block_hash; +} +namespace nano::store +{ +/** + * Manages account storage and iteration + */ +class account +{ +public: + virtual void put (store::write_transaction const &, nano::account const &, nano::account_info const &) = 0; + virtual bool get (store::transaction const &, nano::account const &, nano::account_info &) = 0; + std::optional get (store::transaction const &, nano::account const &); + virtual void del (store::write_transaction const &, nano::account const &) = 0; + virtual bool exists (store::transaction const &, nano::account const &) = 0; + virtual size_t count (store::transaction const &) = 0; + virtual iterator begin (store::transaction const &, nano::account const &) const = 0; + virtual iterator begin (store::transaction const &) const = 0; + virtual iterator rbegin (store::transaction const &) const = 0; + virtual iterator end () const = 0; + virtual void for_each_par (std::function, iterator)> const &) const = 0; +}; +} // namespace nano::store diff --git a/nano/store/block.cpp b/nano/store/block.cpp new file mode 100644 index 0000000000..40d128b96e --- /dev/null +++ b/nano/store/block.cpp @@ -0,0 +1 @@ +#include diff --git a/nano/store/block.hpp b/nano/store/block.hpp new file mode 100644 index 0000000000..826aafc41c --- /dev/null +++ b/nano/store/block.hpp @@ -0,0 +1,42 @@ +#pragma once + +#include +#include +#include +#include + +#include + +namespace nano +{ +class block_hash; +} +namespace nano::store +{ +class block_w_sideband +{ +public: + std::shared_ptr block; + nano::block_sideband sideband; +}; +/** + * Manages block storage and iteration + */ +class block +{ +public: + virtual void put (store::write_transaction const &, nano::block_hash const &, nano::block const &) = 0; + virtual void raw_put (store::write_transaction const &, std::vector const &, nano::block_hash const &) = 0; + virtual nano::block_hash successor (store::transaction const &, nano::block_hash const &) const = 0; + virtual void successor_clear (store::write_transaction const &, nano::block_hash const &) = 0; + virtual std::shared_ptr get (store::transaction const &, nano::block_hash const &) const = 0; + virtual std::shared_ptr random (store::transaction const &) = 0; + virtual void del (store::write_transaction const &, nano::block_hash const &) = 0; + virtual bool exists (store::transaction const &, nano::block_hash const &) = 0; + virtual uint64_t count (store::transaction const &) = 0; + virtual iterator begin (store::transaction const &, nano::block_hash const &) const = 0; + virtual iterator begin (store::transaction const &) const = 0; + virtual iterator end () const = 0; + virtual void for_each_par (std::function, iterator)> const & action_a) const = 0; +}; +} // namespace nano::store diff --git a/nano/store/component.cpp b/nano/store/component.cpp new file mode 100644 index 0000000000..cd68dc2a09 --- /dev/null +++ b/nano/store/component.cpp @@ -0,0 +1,40 @@ +#include +#include +#include +#include +#include +#include + +nano::store::component::component (nano::store::block & block_store_a, nano::store::frontier & frontier_store_a, nano::store::account & account_store_a, nano::store::pending & pending_store_a, nano::store::online_weight & online_weight_store_a, nano::store::pruned & pruned_store_a, nano::store::peer & peer_store_a, nano::store::confirmation_height & confirmation_height_store_a, nano::store::final_vote & final_vote_store_a, nano::store::version & version_store_a) : + block (block_store_a), + frontier (frontier_store_a), + account (account_store_a), + pending (pending_store_a), + online_weight (online_weight_store_a), + pruned (pruned_store_a), + peer (peer_store_a), + confirmation_height (confirmation_height_store_a), + final_vote (final_vote_store_a), + version (version_store_a) +{ +} + +/** + * If using a different store version than the latest then you may need + * to modify some of the objects in the store to be appropriate for the version before an upgrade. + */ +void nano::store::component::initialize (store::write_transaction const & transaction_a, nano::ledger_cache & ledger_cache_a, nano::ledger_constants & constants) +{ + debug_assert (constants.genesis->has_sideband ()); + debug_assert (account.begin (transaction_a) == account.end ()); + auto hash_l (constants.genesis->hash ()); + block.put (transaction_a, hash_l, *constants.genesis); + ++ledger_cache_a.block_count; + confirmation_height.put (transaction_a, constants.genesis->account (), nano::confirmation_height_info{ 1, constants.genesis->hash () }); + ++ledger_cache_a.cemented_count; + ledger_cache_a.final_votes_confirmation_canary = (constants.final_votes_canary_account == constants.genesis->account () && 1 >= constants.final_votes_canary_height); + account.put (transaction_a, constants.genesis->account (), { hash_l, constants.genesis->account (), constants.genesis->hash (), std::numeric_limits::max (), nano::seconds_since_epoch (), 1, nano::epoch::epoch_0 }); + ++ledger_cache_a.account_count; + ledger_cache_a.rep_weights.representation_put (constants.genesis->account (), std::numeric_limits::max ()); + frontier.put (transaction_a, hash_l, constants.genesis->account ()); +} diff --git a/nano/store/component.hpp b/nano/store/component.hpp new file mode 100644 index 0000000000..a6584c9b84 --- /dev/null +++ b/nano/store/component.hpp @@ -0,0 +1,102 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +namespace nano +{ +namespace store +{ + class account; + class block; + class confirmation_height; + class final_vote; + class frontier; + class online_weight; + class peer; + class pending; + class pruned; + class version; +} +class ledger_cache; + +namespace store +{ + /** + * Store manager + */ + class component + { + friend class rocksdb_block_store_tombstone_count_Test; + friend class mdb_block_store_upgrade_v21_v22_Test; + + public: + // clang-format off + explicit component ( + nano::store::block &, + nano::store::frontier &, + nano::store::account &, + nano::store::pending &, + nano::store::online_weight&, + nano::store::pruned &, + nano::store::peer &, + nano::store::confirmation_height &, + nano::store::final_vote &, + nano::store::version & + ); + // clang-format on + virtual ~component () = default; + void initialize (write_transaction const & transaction_a, nano::ledger_cache & ledger_cache_a, nano::ledger_constants & constants); + virtual uint64_t count (store::transaction const & transaction_a, tables table_a) const = 0; + virtual int drop (write_transaction const & transaction_a, tables table_a) = 0; + virtual bool not_found (int status) const = 0; + virtual bool success (int status) const = 0; + virtual int status_code_not_found () const = 0; + virtual std::string error_string (int status) const = 0; + + store::block & block; + store::frontier & frontier; + store::account & account; + store::pending & pending; + static int constexpr version_minimum{ 21 }; + static int constexpr version_current{ 22 }; + + public: + store::online_weight & online_weight; + store::pruned & pruned; + store::peer & peer; + store::confirmation_height & confirmation_height; + store::final_vote & final_vote; + store::version & version; + + virtual unsigned max_block_write_batch_num () const = 0; + + virtual bool copy_db (boost::filesystem::path const & destination) = 0; + virtual void rebuild_db (write_transaction const & transaction_a) = 0; + + /** Not applicable to all sub-classes */ + virtual void serialize_mdb_tracker (boost::property_tree::ptree &, std::chrono::milliseconds, std::chrono::milliseconds){}; + virtual void serialize_memory_stats (boost::property_tree::ptree &) = 0; + + virtual bool init_error () const = 0; + + /** Start read-write transaction */ + virtual write_transaction tx_begin_write (std::vector const & tables_to_lock = {}, std::vector const & tables_no_lock = {}) = 0; + + /** Start read-only transaction */ + virtual read_transaction tx_begin_read () const = 0; + + virtual std::string vendor_get () const = 0; + }; +} // namespace store +} // namespace nano diff --git a/nano/store/confirmation_height.cpp b/nano/store/confirmation_height.cpp new file mode 100644 index 0000000000..dae07d69c7 --- /dev/null +++ b/nano/store/confirmation_height.cpp @@ -0,0 +1,15 @@ +#include + +std::optional nano::store::confirmation_height::get (store::transaction const & transaction, nano::account const & account) +{ + nano::confirmation_height_info info; + bool error = get (transaction, account, info); + if (!error) + { + return info; + } + else + { + return std::nullopt; + } +} diff --git a/nano/store/confirmation_height.hpp b/nano/store/confirmation_height.hpp new file mode 100644 index 0000000000..f0cb5ee73c --- /dev/null +++ b/nano/store/confirmation_height.hpp @@ -0,0 +1,40 @@ +#pragma once + +#include +#include +#include + +#include + +namespace nano +{ +class block_hash; +} +namespace nano::store +{ +/** + * Manages confirmation height storage and iteration + */ +class confirmation_height +{ +public: + virtual void put (store::write_transaction const & transaction_a, nano::account const & account_a, nano::confirmation_height_info const & confirmation_height_info_a) = 0; + + /** Retrieves confirmation height info relating to an account. + * The parameter confirmation_height_info_a is always written. + * On error, the confirmation height and frontier hash are set to 0. + * Ruturns true on error, false on success. + */ + virtual bool get (store::transaction const & transaction_a, nano::account const & account_a, nano::confirmation_height_info & confirmation_height_info_a) = 0; + std::optional get (store::transaction const & transaction_a, nano::account const & account_a); + virtual bool exists (store::transaction const & transaction_a, nano::account const & account_a) const = 0; + virtual void del (store::write_transaction const & transaction_a, nano::account const & account_a) = 0; + virtual uint64_t count (store::transaction const & transaction_a) = 0; + virtual void clear (store::write_transaction const &, nano::account const &) = 0; + virtual void clear (store::write_transaction const &) = 0; + virtual iterator begin (store::transaction const & transaction_a, nano::account const & account_a) const = 0; + virtual iterator begin (store::transaction const & transaction_a) const = 0; + virtual iterator end () const = 0; + virtual void for_each_par (std::function, iterator)> const &) const = 0; +}; +} // namespace nano::store diff --git a/nano/store/db_val.cpp b/nano/store/db_val.cpp new file mode 100644 index 0000000000..cab8d8ef04 --- /dev/null +++ b/nano/store/db_val.cpp @@ -0,0 +1 @@ +#include diff --git a/nano/store/db_val.hpp b/nano/store/db_val.hpp new file mode 100644 index 0000000000..957492e23e --- /dev/null +++ b/nano/store/db_val.hpp @@ -0,0 +1,325 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +namespace nano::store +{ +/** + * Encapsulates database specific container + */ +template +class db_val +{ +public: + db_val (Val const & value_a) : + value (value_a) + { + } + + db_val () : + db_val (0, nullptr) + { + } + + db_val (std::nullptr_t) : + db_val (0, this) + { + } + + db_val (nano::uint128_union const & val_a) : + db_val (sizeof (val_a), const_cast (&val_a)) + { + } + + db_val (nano::uint256_union const & val_a) : + db_val (sizeof (val_a), const_cast (&val_a)) + { + } + + db_val (nano::uint512_union const & val_a) : + db_val (sizeof (val_a), const_cast (&val_a)) + { + } + + db_val (nano::qualified_root const & val_a) : + db_val (sizeof (val_a), const_cast (&val_a)) + { + } + + db_val (nano::account_info const & val_a) : + db_val (val_a.db_size (), const_cast (&val_a)) + { + } + + db_val (nano::pending_info const & val_a) : + db_val (val_a.db_size (), const_cast (&val_a)) + { + static_assert (std::is_standard_layout::value, "Standard layout is required"); + } + + db_val (nano::pending_key const & val_a) : + db_val (sizeof (val_a), const_cast (&val_a)) + { + static_assert (std::is_standard_layout::value, "Standard layout is required"); + } + + db_val (nano::confirmation_height_info const & val_a) : + buffer (std::make_shared> ()) + { + { + nano::vectorstream stream (*buffer); + val_a.serialize (stream); + } + convert_buffer_to_value (); + } + + db_val (nano::block_info const & val_a) : + db_val (sizeof (val_a), const_cast (&val_a)) + { + static_assert (std::is_standard_layout::value, "Standard layout is required"); + } + + db_val (nano::endpoint_key const & val_a) : + db_val (sizeof (val_a), const_cast (&val_a)) + { + static_assert (std::is_standard_layout::value, "Standard layout is required"); + } + + db_val (std::shared_ptr const & val_a) : + buffer (std::make_shared> ()) + { + { + nano::vectorstream stream (*buffer); + nano::serialize_block (stream, *val_a); + } + convert_buffer_to_value (); + } + + db_val (uint64_t val_a) : + buffer (std::make_shared> ()) + { + { + boost::endian::native_to_big_inplace (val_a); + nano::vectorstream stream (*buffer); + nano::write (stream, val_a); + } + convert_buffer_to_value (); + } + + explicit operator nano::account_info () const + { + nano::account_info result; + debug_assert (size () == result.db_size ()); + std::copy (reinterpret_cast (data ()), reinterpret_cast (data ()) + result.db_size (), reinterpret_cast (&result)); + return result; + } + + explicit operator block_info () const + { + nano::block_info result; + debug_assert (size () == sizeof (result)); + static_assert (sizeof (nano::block_info::account) + sizeof (nano::block_info::balance) == sizeof (result), "Packed class"); + std::copy (reinterpret_cast (data ()), reinterpret_cast (data ()) + sizeof (result), reinterpret_cast (&result)); + return result; + } + + explicit operator nano::pending_info () const + { + nano::pending_info result; + debug_assert (size () == result.db_size ()); + std::copy (reinterpret_cast (data ()), reinterpret_cast (data ()) + result.db_size (), reinterpret_cast (&result)); + return result; + } + + explicit operator nano::pending_key () const + { + nano::pending_key result; + debug_assert (size () == sizeof (result)); + static_assert (sizeof (nano::pending_key::account) + sizeof (nano::pending_key::hash) == sizeof (result), "Packed class"); + std::copy (reinterpret_cast (data ()), reinterpret_cast (data ()) + sizeof (result), reinterpret_cast (&result)); + return result; + } + + explicit operator nano::confirmation_height_info () const + { + nano::bufferstream stream (reinterpret_cast (data ()), size ()); + nano::confirmation_height_info result; + bool error (result.deserialize (stream)); + (void)error; + debug_assert (!error); + return result; + } + + explicit operator nano::uint128_union () const + { + return convert (); + } + + explicit operator nano::amount () const + { + return convert (); + } + + explicit operator nano::block_hash () const + { + return convert (); + } + + explicit operator nano::public_key () const + { + return convert (); + } + + explicit operator nano::qualified_root () const + { + return convert (); + } + + explicit operator nano::uint256_union () const + { + return convert (); + } + + explicit operator nano::uint512_union () const + { + return convert (); + } + + explicit operator std::array () const + { + nano::bufferstream stream (reinterpret_cast (data ()), size ()); + std::array result; + auto error = nano::try_read (stream, result); + (void)error; + debug_assert (!error); + return result; + } + + explicit operator nano::endpoint_key () const + { + nano::endpoint_key result; + std::copy (reinterpret_cast (data ()), reinterpret_cast (data ()) + sizeof (result), reinterpret_cast (&result)); + return result; + } + + explicit operator block_w_sideband () const + { + nano::bufferstream stream (reinterpret_cast (data ()), size ()); + nano::store::block_w_sideband block_w_sideband; + block_w_sideband.block = (nano::deserialize_block (stream)); + auto error = block_w_sideband.sideband.deserialize (stream, block_w_sideband.block->type ()); + release_assert (!error); + block_w_sideband.block->sideband_set (block_w_sideband.sideband); + return block_w_sideband; + } + + explicit operator std::nullptr_t () const + { + return nullptr; + } + + explicit operator nano::no_value () const + { + return no_value::dummy; + } + + explicit operator std::shared_ptr () const + { + nano::bufferstream stream (reinterpret_cast (data ()), size ()); + std::shared_ptr result (nano::deserialize_block (stream)); + return result; + } + + template + std::shared_ptr convert_to_block () const + { + nano::bufferstream stream (reinterpret_cast (data ()), size ()); + auto error (false); + auto result (std::make_shared (error, stream)); + debug_assert (!error); + return result; + } + + explicit operator std::shared_ptr () const + { + return convert_to_block (); + } + + explicit operator std::shared_ptr () const + { + return convert_to_block (); + } + + explicit operator std::shared_ptr () const + { + return convert_to_block (); + } + + explicit operator std::shared_ptr () const + { + return convert_to_block (); + } + + explicit operator std::shared_ptr () const + { + return convert_to_block (); + } + + explicit operator std::shared_ptr () const + { + nano::bufferstream stream (reinterpret_cast (data ()), size ()); + auto error (false); + auto result (nano::make_shared (error, stream)); + debug_assert (!error); + return result; + } + + explicit operator uint64_t () const + { + uint64_t result; + nano::bufferstream stream (reinterpret_cast (data ()), size ()); + auto error (nano::try_read (stream, result)); + (void)error; + debug_assert (!error); + boost::endian::big_to_native_inplace (result); + return result; + } + + operator Val * () const + { + // Allow passing a temporary to a non-c++ function which doesn't have constness + return const_cast (&value); + } + + operator Val const & () const + { + return value; + } + + // Must be specialized + void * data () const; + size_t size () const; + db_val (size_t size_a, void * data_a); + void convert_buffer_to_value (); + + Val value; + std::shared_ptr> buffer; + +private: + template + T convert () const + { + T result; + debug_assert (size () == sizeof (result)); + std::copy (reinterpret_cast (data ()), reinterpret_cast (data ()) + sizeof (result), result.bytes.data ()); + return result; + } +}; +} // namespace nano::store diff --git a/nano/store/final.cpp b/nano/store/final.cpp new file mode 100644 index 0000000000..69d469dc76 --- /dev/null +++ b/nano/store/final.cpp @@ -0,0 +1 @@ +#include diff --git a/nano/store/final.hpp b/nano/store/final.hpp new file mode 100644 index 0000000000..1b9f159951 --- /dev/null +++ b/nano/store/final.hpp @@ -0,0 +1,32 @@ +#pragma once + +#include +#include +#include + +#include + +namespace nano +{ +class block_hash; +} +namespace nano::store +{ +/** + * Manages final vote storage and iteration + */ +class final_vote +{ +public: + virtual bool put (store::write_transaction const & transaction_a, nano::qualified_root const & root_a, nano::block_hash const & hash_a) = 0; + virtual std::vector get (store::transaction const & transaction_a, nano::root const & root_a) = 0; + virtual void del (store::write_transaction const & transaction_a, nano::root const & root_a) = 0; + virtual size_t count (store::transaction const & transaction_a) const = 0; + virtual void clear (store::write_transaction const &, nano::root const &) = 0; + virtual void clear (store::write_transaction const &) = 0; + virtual store::iterator begin (store::transaction const & transaction_a, nano::qualified_root const & root_a) const = 0; + virtual store::iterator begin (store::transaction const & transaction_a) const = 0; + virtual store::iterator end () const = 0; + virtual void for_each_par (std::function, store::iterator)> const & action_a) const = 0; +}; +} // namespace nano::store diff --git a/nano/store/frontier.cpp b/nano/store/frontier.cpp new file mode 100644 index 0000000000..2d12551d59 --- /dev/null +++ b/nano/store/frontier.cpp @@ -0,0 +1 @@ +#include diff --git a/nano/store/frontier.hpp b/nano/store/frontier.hpp new file mode 100644 index 0000000000..b4d504d416 --- /dev/null +++ b/nano/store/frontier.hpp @@ -0,0 +1,29 @@ +#pragma once + +#include +#include +#include + +#include + +namespace nano +{ +class block_hash; +} +namespace nano::store +{ +/** + * Manages frontier storage and iteration + */ +class frontier +{ +public: + virtual void put (store::write_transaction const &, nano::block_hash const &, nano::account const &) = 0; + virtual nano::account get (store::transaction const &, nano::block_hash const &) const = 0; + virtual void del (store::write_transaction const &, nano::block_hash const &) = 0; + virtual iterator begin (store::transaction const &) const = 0; + virtual iterator begin (store::transaction const &, nano::block_hash const &) const = 0; + virtual iterator end () const = 0; + virtual void for_each_par (std::function, store::iterator)> const & action_a) const = 0; +}; +} // namespace nano::store diff --git a/nano/store/iterator.cpp b/nano/store/iterator.cpp new file mode 100644 index 0000000000..15bf18facf --- /dev/null +++ b/nano/store/iterator.cpp @@ -0,0 +1 @@ +#include diff --git a/nano/store/iterator.hpp b/nano/store/iterator.hpp new file mode 100644 index 0000000000..00f2ab7f93 --- /dev/null +++ b/nano/store/iterator.hpp @@ -0,0 +1,65 @@ +#pragma once + +#include + +#include + +namespace nano::store +{ +/** + * Iterates the key/value pairs of a transaction + */ +template +class iterator final +{ +public: + iterator (std::nullptr_t) + { + } + iterator (std::unique_ptr> impl_a) : + impl (std::move (impl_a)) + { + impl->fill (current); + } + iterator (iterator && other_a) : + current (std::move (other_a.current)), + impl (std::move (other_a.impl)) + { + } + iterator & operator++ () + { + ++*impl; + impl->fill (current); + return *this; + } + iterator & operator-- () + { + --*impl; + impl->fill (current); + return *this; + } + iterator & operator= (iterator && other_a) noexcept + { + impl = std::move (other_a.impl); + current = std::move (other_a.current); + return *this; + } + iterator & operator= (iterator const &) = delete; + std::pair * operator-> () + { + return ¤t; + } + bool operator== (iterator const & other_a) const + { + return (impl == nullptr && other_a.impl == nullptr) || (impl != nullptr && *impl == other_a.impl.get ()) || (other_a.impl != nullptr && *other_a.impl == impl.get ()); + } + bool operator!= (iterator const & other_a) const + { + return !(*this == other_a); + } + +private: + std::pair current; + std::unique_ptr> impl; +}; +} // namespace nano::store diff --git a/nano/store/iterator_impl.cpp b/nano/store/iterator_impl.cpp new file mode 100644 index 0000000000..4abbeb3f2a --- /dev/null +++ b/nano/store/iterator_impl.cpp @@ -0,0 +1 @@ +#include diff --git a/nano/store/iterator_impl.hpp b/nano/store/iterator_impl.hpp new file mode 100644 index 0000000000..0f873e976b --- /dev/null +++ b/nano/store/iterator_impl.hpp @@ -0,0 +1,27 @@ +#pragma once + +#include + +namespace nano::store +{ +template +class iterator_impl +{ +public: + virtual ~iterator_impl () = default; + virtual iterator_impl & operator++ () = 0; + virtual iterator_impl & operator-- () = 0; + virtual bool operator== (iterator_impl const & other_a) const = 0; + virtual bool is_end_sentinal () const = 0; + virtual void fill (std::pair &) const = 0; + iterator_impl & operator= (iterator_impl const &) = delete; + bool operator== (iterator_impl const * other_a) const + { + return (other_a != nullptr && *this == *other_a) || (other_a == nullptr && is_end_sentinal ()); + } + bool operator!= (iterator_impl const & other_a) const + { + return !(*this == other_a); + } +}; +} // namespace nano::store diff --git a/nano/store/lmdb/account.cpp b/nano/store/lmdb/account.cpp new file mode 100644 index 0000000000..19a35c8e44 --- /dev/null +++ b/nano/store/lmdb/account.cpp @@ -0,0 +1,73 @@ +#include +#include +#include +#include + +nano::store::lmdb::account::account (nano::store::lmdb::component & store_a) : + store (store_a){}; + +void nano::store::lmdb::account::put (store::write_transaction const & transaction, nano::account const & account, nano::account_info const & info) +{ + auto status = store.put (transaction, tables::accounts, account, info); + store.release_assert_success (status); +} + +bool nano::store::lmdb::account::get (store::transaction const & transaction, nano::account const & account, nano::account_info & info) +{ + nano::store::lmdb::db_val value; + auto status1 (store.get (transaction, tables::accounts, account, value)); + release_assert (store.success (status1) || store.not_found (status1)); + bool result (true); + if (store.success (status1)) + { + nano::bufferstream stream (reinterpret_cast (value.data ()), value.size ()); + result = info.deserialize (stream); + } + return result; +} + +void nano::store::lmdb::account::del (store::write_transaction const & transaction_a, nano::account const & account_a) +{ + auto status = store.del (transaction_a, tables::accounts, account_a); + store.release_assert_success (status); +} + +bool nano::store::lmdb::account::exists (store::transaction const & transaction_a, nano::account const & account_a) +{ + auto iterator (begin (transaction_a, account_a)); + return iterator != end () && nano::account (iterator->first) == account_a; +} + +size_t nano::store::lmdb::account::count (store::transaction const & transaction_a) +{ + return store.count (transaction_a, tables::accounts); +} + +nano::store::iterator nano::store::lmdb::account::begin (store::transaction const & transaction, nano::account const & account) const +{ + return store.make_iterator (transaction, tables::accounts, account); +} + +nano::store::iterator nano::store::lmdb::account::begin (store::transaction const & transaction) const +{ + return store.make_iterator (transaction, tables::accounts); +} + +nano::store::iterator nano::store::lmdb::account::rbegin (store::transaction const & transaction_a) const +{ + return store.make_iterator (transaction_a, tables::accounts, false); +} + +nano::store::iterator nano::store::lmdb::account::end () const +{ + return store::iterator (nullptr); +} + +void nano::store::lmdb::account::for_each_par (std::function, store::iterator)> const & action_a) const +{ + parallel_traversal ( + [&action_a, this] (nano::uint256_t const & start, nano::uint256_t const & end, bool const is_last) { + auto transaction (this->store.tx_begin_read ()); + action_a (transaction, this->begin (transaction, start), !is_last ? this->begin (transaction, end) : this->end ()); + }); +} diff --git a/nano/store/lmdb/account.hpp b/nano/store/lmdb/account.hpp new file mode 100644 index 0000000000..9c3284b482 --- /dev/null +++ b/nano/store/lmdb/account.hpp @@ -0,0 +1,55 @@ +#pragma once + +#include + +#include + +namespace nano::store::lmdb +{ +class component; +} +namespace nano::store::lmdb +{ +class account : public nano::store::account +{ +private: + nano::store::lmdb::component & store; + +public: + explicit account (nano::store::lmdb::component & store_a); + void put (store::write_transaction const & transaction, nano::account const & account, nano::account_info const & info) override; + bool get (store::transaction const & transaction_a, nano::account const & account_a, nano::account_info & info_a) override; + void del (store::write_transaction const & transaction_a, nano::account const & account_a) override; + bool exists (store::transaction const & transaction_a, nano::account const & account_a) override; + size_t count (store::transaction const & transaction_a) override; + store::iterator begin (store::transaction const & transaction_a, nano::account const & account_a) const override; + store::iterator begin (store::transaction const & transaction_a) const override; + store::iterator rbegin (store::transaction const & transaction_a) const override; + store::iterator end () const override; + void for_each_par (std::function, store::iterator)> const & action_a) const override; + + /** + * Maps account v1 to account information, head, rep, open, balance, timestamp and block count. (Removed) + * nano::account -> nano::block_hash, nano::block_hash, nano::block_hash, nano::amount, uint64_t, uint64_t + */ + MDB_dbi accounts_v0_handle{ 0 }; + + /** + * Maps account v0 to account information, head, rep, open, balance, timestamp and block count. (Removed) + * nano::account -> nano::block_hash, nano::block_hash, nano::block_hash, nano::amount, uint64_t, uint64_t + */ + MDB_dbi accounts_v1_handle{ 0 }; + + /** + * Maps account v0 to account information, head, rep, open, balance, timestamp, block count and epoch + * nano::account -> nano::block_hash, nano::block_hash, nano::block_hash, nano::amount, uint64_t, uint64_t, nano::epoch + */ + MDB_dbi accounts_handle{ 0 }; + + /** + * Representative weights. (Removed) + * nano::account -> nano::uint128_t + */ + MDB_dbi representation_handle{ 0 }; +}; +} // amespace nano::store::lmdb diff --git a/nano/store/lmdb/block.cpp b/nano/store/lmdb/block.cpp new file mode 100644 index 0000000000..e01d71fb21 --- /dev/null +++ b/nano/store/lmdb/block.cpp @@ -0,0 +1,213 @@ +#include +#include +#include + +namespace nano::store::lmdb +{ +/** + * Fill in our predecessors + */ +class block_predecessor_mdb_set : public nano::block_visitor +{ +public: + block_predecessor_mdb_set (store::write_transaction const & transaction_a, nano::store::lmdb::block & block_store_a); + virtual ~block_predecessor_mdb_set () = default; + void fill_value (nano::block const & block_a); + void send_block (nano::send_block const & block_a) override; + void receive_block (nano::receive_block const & block_a) override; + void open_block (nano::open_block const & block_a) override; + void change_block (nano::change_block const & block_a) override; + void state_block (nano::state_block const & block_a) override; + store::write_transaction const & transaction; + nano::store::lmdb::block & block_store; +}; +} + +nano::store::lmdb::block::block (nano::store::lmdb::component & store_a) : + store{ store_a } {}; + +void nano::store::lmdb::block::put (store::write_transaction const & transaction, nano::block_hash const & hash, nano::block const & block) +{ + debug_assert (block.sideband ().successor.is_zero () || exists (transaction, block.sideband ().successor)); + std::vector vector; + { + nano::vectorstream stream (vector); + nano::serialize_block (stream, block); + block.sideband ().serialize (stream, block.type ()); + } + raw_put (transaction, vector, hash); + block_predecessor_mdb_set predecessor (transaction, *this); + block.visit (predecessor); + debug_assert (block.previous ().is_zero () || successor (transaction, block.previous ()) == hash); +} + +void nano::store::lmdb::block::raw_put (store::write_transaction const & transaction_a, std::vector const & data, nano::block_hash const & hash_a) +{ + nano::store::lmdb::db_val value{ data.size (), (void *)data.data () }; + auto status = store.put (transaction_a, tables::blocks, hash_a, value); + store.release_assert_success (status); +} + +nano::block_hash nano::store::lmdb::block::successor (store::transaction const & transaction_a, nano::block_hash const & hash_a) const +{ + nano::store::lmdb::db_val value; + block_raw_get (transaction_a, hash_a, value); + nano::block_hash result; + if (value.size () != 0) + { + debug_assert (value.size () >= result.bytes.size ()); + auto type = block_type_from_raw (value.data ()); + nano::bufferstream stream (reinterpret_cast (value.data ()) + block_successor_offset (transaction_a, value.size (), type), result.bytes.size ()); + auto error (nano::try_read (stream, result.bytes)); + (void)error; + debug_assert (!error); + } + else + { + result.clear (); + } + return result; +} + +void nano::store::lmdb::block::successor_clear (store::write_transaction const & transaction, nano::block_hash const & hash) +{ + nano::store::lmdb::db_val value; + block_raw_get (transaction, hash, value); + debug_assert (value.size () != 0); + auto type = block_type_from_raw (value.data ()); + std::vector data (static_cast (value.data ()), static_cast (value.data ()) + value.size ()); + std::fill_n (data.begin () + block_successor_offset (transaction, value.size (), type), sizeof (nano::block_hash), uint8_t{ 0 }); + raw_put (transaction, data, hash); +} + +std::shared_ptr nano::store::lmdb::block::get (store::transaction const & transaction, nano::block_hash const & hash) const +{ + nano::store::lmdb::db_val value; + block_raw_get (transaction, hash, value); + std::shared_ptr result; + if (value.size () != 0) + { + nano::bufferstream stream (reinterpret_cast (value.data ()), value.size ()); + nano::block_type type; + auto error (try_read (stream, type)); + release_assert (!error); + result = nano::deserialize_block (stream, type); + release_assert (result != nullptr); + nano::block_sideband sideband; + error = (sideband.deserialize (stream, type)); + release_assert (!error); + result->sideband_set (sideband); + } + return result; +} + +std::shared_ptr nano::store::lmdb::block::random (store::transaction const & transaction) +{ + nano::block_hash hash; + nano::random_pool::generate_block (hash.bytes.data (), hash.bytes.size ()); + auto existing = begin (transaction, hash); + if (existing == end ()) + { + existing = begin (transaction); + } + debug_assert (existing != end ()); + return existing->second.block; +} + +void nano::store::lmdb::block::del (store::write_transaction const & transaction_a, nano::block_hash const & hash_a) +{ + auto status = store.del (transaction_a, tables::blocks, hash_a); + store.release_assert_success (status); +} + +bool nano::store::lmdb::block::exists (store::transaction const & transaction, nano::block_hash const & hash) +{ + nano::store::lmdb::db_val junk; + block_raw_get (transaction, hash, junk); + return junk.size () != 0; +} + +uint64_t nano::store::lmdb::block::count (store::transaction const & transaction_a) +{ + return store.count (transaction_a, tables::blocks); +} +nano::store::iterator nano::store::lmdb::block::begin (store::transaction const & transaction) const +{ + return store.make_iterator (transaction, tables::blocks); +} + +nano::store::iterator nano::store::lmdb::block::begin (store::transaction const & transaction, nano::block_hash const & hash) const +{ + return store.make_iterator (transaction, tables::blocks, hash); +} + +nano::store::iterator nano::store::lmdb::block::end () const +{ + return store::iterator (nullptr); +} + +void nano::store::lmdb::block::for_each_par (std::function, store::iterator)> const & action_a) const +{ + parallel_traversal ( + [&action_a, this] (nano::uint256_t const & start, nano::uint256_t const & end, bool const is_last) { + auto transaction (this->store.tx_begin_read ()); + action_a (transaction, this->begin (transaction, start), !is_last ? this->begin (transaction, end) : this->end ()); + }); +} + +void nano::store::lmdb::block::block_raw_get (store::transaction const & transaction, nano::block_hash const & hash, nano::store::lmdb::db_val & value) const +{ + auto status = store.get (transaction, tables::blocks, hash, value); + release_assert (store.success (status) || store.not_found (status)); +} + +size_t nano::store::lmdb::block::block_successor_offset (store::transaction const & transaction_a, size_t entry_size_a, nano::block_type type_a) const +{ + return entry_size_a - nano::block_sideband::size (type_a); +} + +nano::block_type nano::store::lmdb::block::block_type_from_raw (void * data_a) +{ + // The block type is the first byte + return static_cast ((reinterpret_cast (data_a))[0]); +} + +nano::store::lmdb::block_predecessor_mdb_set::block_predecessor_mdb_set (store::write_transaction const & transaction_a, nano::store::lmdb::block & block_store_a) : + transaction{ transaction_a }, + block_store{ block_store_a } +{ +} +void nano::store::lmdb::block_predecessor_mdb_set::fill_value (nano::block const & block_a) +{ + auto hash = block_a.hash (); + nano::store::lmdb::db_val value; + block_store.block_raw_get (transaction, block_a.previous (), value); + debug_assert (value.size () != 0); + auto type = block_store.block_type_from_raw (value.data ()); + std::vector data (static_cast (value.data ()), static_cast (value.data ()) + value.size ()); + std::copy (hash.bytes.begin (), hash.bytes.end (), data.begin () + block_store.block_successor_offset (transaction, value.size (), type)); + block_store.raw_put (transaction, data, block_a.previous ()); +} +void nano::store::lmdb::block_predecessor_mdb_set::send_block (nano::send_block const & block_a) +{ + fill_value (block_a); +} +void nano::store::lmdb::block_predecessor_mdb_set::receive_block (nano::receive_block const & block_a) +{ + fill_value (block_a); +} +void nano::store::lmdb::block_predecessor_mdb_set::open_block (nano::open_block const & block_a) +{ + // Open blocks don't have a predecessor +} +void nano::store::lmdb::block_predecessor_mdb_set::change_block (nano::change_block const & block_a) +{ + fill_value (block_a); +} +void nano::store::lmdb::block_predecessor_mdb_set::state_block (nano::state_block const & block_a) +{ + if (!block_a.previous ().is_zero ()) + { + fill_value (block_a); + } +} diff --git a/nano/store/lmdb/block.hpp b/nano/store/lmdb/block.hpp new file mode 100644 index 0000000000..a16f213859 --- /dev/null +++ b/nano/store/lmdb/block.hpp @@ -0,0 +1,50 @@ +#pragma once + +#include +#include + +#include + +namespace nano::store::lmdb +{ +class block_predecessor_mdb_set; +} +namespace nano::store::lmdb +{ +class component; +} +namespace nano::store::lmdb +{ +class block : public nano::store::block +{ + friend class block_predecessor_mdb_set; + nano::store::lmdb::component & store; + +public: + explicit block (nano::store::lmdb::component & store_a); + void put (store::write_transaction const & transaction_a, nano::block_hash const & hash_a, nano::block const & block_a) override; + void raw_put (store::write_transaction const & transaction_a, std::vector const & data, nano::block_hash const & hash_a) override; + nano::block_hash successor (store::transaction const & transaction_a, nano::block_hash const & hash_a) const override; + void successor_clear (store::write_transaction const & transaction_a, nano::block_hash const & hash_a) override; + std::shared_ptr get (store::transaction const & transaction_a, nano::block_hash const & hash_a) const override; + std::shared_ptr random (store::transaction const & transaction_a) override; + void del (store::write_transaction const & transaction_a, nano::block_hash const & hash_a) override; + bool exists (store::transaction const & transaction_a, nano::block_hash const & hash_a) override; + uint64_t count (store::transaction const & transaction_a) override; + store::iterator begin (store::transaction const & transaction_a) const override; + store::iterator begin (store::transaction const & transaction_a, nano::block_hash const & hash_a) const override; + store::iterator end () const override; + void for_each_par (std::function, store::iterator)> const & action_a) const override; + + /** + * Contains block_sideband and block for all block types (legacy send/change/open/receive & state blocks) + * nano::block_hash -> nano::block_sideband, nano::block + */ + MDB_dbi blocks_handle{ 0 }; + +protected: + void block_raw_get (store::transaction const & transaction_a, nano::block_hash const & hash_a, db_val & value) const; + size_t block_successor_offset (store::transaction const & transaction_a, size_t entry_size_a, nano::block_type type_a) const; + static nano::block_type block_type_from_raw (void * data_a); +}; +} // namespace nano::store::lmdb diff --git a/nano/store/lmdb/confirmation_height.cpp b/nano/store/lmdb/confirmation_height.cpp new file mode 100644 index 0000000000..1ce01edab4 --- /dev/null +++ b/nano/store/lmdb/confirmation_height.cpp @@ -0,0 +1,84 @@ +#include +#include +#include + +nano::store::lmdb::confirmation_height::confirmation_height (nano::store::lmdb::component & store) : + store{ store } +{ +} + +void nano::store::lmdb::confirmation_height::put (store::write_transaction const & transaction, nano::account const & account, nano::confirmation_height_info const & confirmation_height_info) +{ + auto status = store.put (transaction, tables::confirmation_height, account, confirmation_height_info); + store.release_assert_success (status); +} + +bool nano::store::lmdb::confirmation_height::get (store::transaction const & transaction, nano::account const & account, nano::confirmation_height_info & confirmation_height_info) +{ + nano::store::lmdb::db_val value; + auto status = store.get (transaction, tables::confirmation_height, account, value); + release_assert (store.success (status) || store.not_found (status)); + bool result (true); + if (store.success (status)) + { + nano::bufferstream stream (reinterpret_cast (value.data ()), value.size ()); + result = confirmation_height_info.deserialize (stream); + } + if (result) + { + confirmation_height_info.height = 0; + confirmation_height_info.frontier = 0; + } + + return result; +} + +bool nano::store::lmdb::confirmation_height::exists (store::transaction const & transaction, nano::account const & account) const +{ + return store.exists (transaction, tables::confirmation_height, account); +} + +void nano::store::lmdb::confirmation_height::del (store::write_transaction const & transaction, nano::account const & account) +{ + auto status = store.del (transaction, tables::confirmation_height, account); + store.release_assert_success (status); +} + +uint64_t nano::store::lmdb::confirmation_height::count (store::transaction const & transaction_a) +{ + return store.count (transaction_a, tables::confirmation_height); +} + +void nano::store::lmdb::confirmation_height::clear (store::write_transaction const & transaction_a, nano::account const & account_a) +{ + del (transaction_a, account_a); +} + +void nano::store::lmdb::confirmation_height::clear (store::write_transaction const & transaction_a) +{ + store.drop (transaction_a, nano::tables::confirmation_height); +} + +nano::store::iterator nano::store::lmdb::confirmation_height::begin (store::transaction const & transaction, nano::account const & account) const +{ + return store.make_iterator (transaction, tables::confirmation_height, account); +} + +nano::store::iterator nano::store::lmdb::confirmation_height::begin (store::transaction const & transaction) const +{ + return store.make_iterator (transaction, tables::confirmation_height); +} + +nano::store::iterator nano::store::lmdb::confirmation_height::end () const +{ + return store::iterator (nullptr); +} + +void nano::store::lmdb::confirmation_height::for_each_par (std::function, store::iterator)> const & action_a) const +{ + parallel_traversal ( + [&action_a, this] (nano::uint256_t const & start, nano::uint256_t const & end, bool const is_last) { + auto transaction (this->store.tx_begin_read ()); + action_a (transaction, this->begin (transaction, start), !is_last ? this->begin (transaction, end) : this->end ()); + }); +} diff --git a/nano/store/lmdb/confirmation_height.hpp b/nano/store/lmdb/confirmation_height.hpp new file mode 100644 index 0000000000..738c8482c6 --- /dev/null +++ b/nano/store/lmdb/confirmation_height.hpp @@ -0,0 +1,37 @@ +#pragma once + +#include + +#include + +namespace nano::store::lmdb +{ +class component; +} +namespace nano::store::lmdb +{ +class confirmation_height : public nano::store::confirmation_height +{ + nano::store::lmdb::component & store; + +public: + explicit confirmation_height (nano::store::lmdb::component & store_a); + void put (store::write_transaction const & transaction_a, nano::account const & account_a, nano::confirmation_height_info const & confirmation_height_info_a) override; + bool get (store::transaction const & transaction_a, nano::account const & account_a, nano::confirmation_height_info & confirmation_height_info_a) override; + bool exists (store::transaction const & transaction_a, nano::account const & account_a) const override; + void del (store::write_transaction const & transaction_a, nano::account const & account_a) override; + uint64_t count (store::transaction const & transaction_a) override; + void clear (store::write_transaction const & transaction_a, nano::account const & account_a) override; + void clear (store::write_transaction const & transaction_a) override; + store::iterator begin (store::transaction const & transaction_a, nano::account const & account_a) const override; + store::iterator begin (store::transaction const & transaction_a) const override; + store::iterator end () const override; + void for_each_par (std::function, store::iterator)> const & action_a) const override; + + /* + * Confirmation height of an account, and the hash for the block at that height + * nano::account -> uint64_t, nano::block_hash + */ + MDB_dbi confirmation_height_handle{ 0 }; +}; +} // namespace nano::store::lmdb diff --git a/nano/store/lmdb/db_val.cpp b/nano/store/lmdb/db_val.cpp new file mode 100644 index 0000000000..7896bbaa5e --- /dev/null +++ b/nano/store/lmdb/db_val.cpp @@ -0,0 +1,25 @@ +#include + +template <> +void * nano::store::lmdb::db_val::data () const +{ + return value.mv_data; +} + +template <> +std::size_t nano::store::lmdb::db_val::size () const +{ + return value.mv_size; +} + +template <> +nano::store::lmdb::db_val::db_val (std::size_t size_a, void * data_a) : + value ({ size_a, data_a }) +{ +} + +template <> +void nano::store::lmdb::db_val::convert_buffer_to_value () +{ + value = { buffer->size (), const_cast (buffer->data ()) }; +} diff --git a/nano/store/lmdb/db_val.hpp b/nano/store/lmdb/db_val.hpp new file mode 100644 index 0000000000..3b4966f5c7 --- /dev/null +++ b/nano/store/lmdb/db_val.hpp @@ -0,0 +1,10 @@ +#pragma once + +#include + +#include + +namespace nano::store::lmdb +{ +using db_val = store::db_val; +} diff --git a/nano/node/rocksdb/final_vote_store.cpp b/nano/store/lmdb/final_vote.cpp similarity index 55% rename from nano/node/rocksdb/final_vote_store.cpp rename to nano/store/lmdb/final_vote.cpp index 6809425ed2..2ae93bd5a9 100644 --- a/nano/node/rocksdb/final_vote_store.cpp +++ b/nano/store/lmdb/final_vote.cpp @@ -1,13 +1,13 @@ -#include -#include #include +#include +#include -nano::rocksdb::final_vote_store::final_vote_store (nano::rocksdb::store & store) : +nano::store::lmdb::final_vote::final_vote (nano::store::lmdb::component & store) : store{ store } {}; -bool nano::rocksdb::final_vote_store::put (nano::write_transaction const & transaction, nano::qualified_root const & root, nano::block_hash const & hash) +bool nano::store::lmdb::final_vote::put (store::write_transaction const & transaction, nano::qualified_root const & root, nano::block_hash const & hash) { - nano::rocksdb_val value; + nano::store::lmdb::db_val value; auto status = store.get (transaction, tables::final_votes, root, value); release_assert (store.success (status) || store.not_found (status)); bool result (true); @@ -23,7 +23,7 @@ bool nano::rocksdb::final_vote_store::put (nano::write_transaction const & trans return result; } -std::vector nano::rocksdb::final_vote_store::get (nano::transaction const & transaction, nano::root const & root_a) +std::vector nano::store::lmdb::final_vote::get (store::transaction const & transaction, nano::root const & root_a) { std::vector result; nano::qualified_root key_start{ root_a.raw, 0 }; @@ -34,7 +34,7 @@ std::vector nano::rocksdb::final_vote_store::get (nano::transa return result; } -void nano::rocksdb::final_vote_store::del (nano::write_transaction const & transaction, nano::root const & root) +void nano::store::lmdb::final_vote::del (store::write_transaction const & transaction, nano::root const & root) { std::vector final_vote_qualified_roots; for (auto i = begin (transaction, nano::qualified_root{ root.raw, 0 }), n = end (); i != n && nano::qualified_root{ i->first }.root () == root; ++i) @@ -49,37 +49,37 @@ void nano::rocksdb::final_vote_store::del (nano::write_transaction const & trans } } -size_t nano::rocksdb::final_vote_store::count (nano::transaction const & transaction_a) const +size_t nano::store::lmdb::final_vote::count (store::transaction const & transaction_a) const { return store.count (transaction_a, tables::final_votes); } -void nano::rocksdb::final_vote_store::clear (nano::write_transaction const & transaction_a, nano::root const & root_a) +void nano::store::lmdb::final_vote::clear (store::write_transaction const & transaction_a, nano::root const & root_a) { del (transaction_a, root_a); } -void nano::rocksdb::final_vote_store::clear (nano::write_transaction const & transaction_a) +void nano::store::lmdb::final_vote::clear (store::write_transaction const & transaction_a) { store.drop (transaction_a, nano::tables::final_votes); } -nano::store_iterator nano::rocksdb::final_vote_store::begin (nano::transaction const & transaction, nano::qualified_root const & root) const +nano::store::iterator nano::store::lmdb::final_vote::begin (store::transaction const & transaction, nano::qualified_root const & root) const { return store.make_iterator (transaction, tables::final_votes, root); } -nano::store_iterator nano::rocksdb::final_vote_store::begin (nano::transaction const & transaction) const +nano::store::iterator nano::store::lmdb::final_vote::begin (store::transaction const & transaction) const { return store.make_iterator (transaction, tables::final_votes); } -nano::store_iterator nano::rocksdb::final_vote_store::end () const +nano::store::iterator nano::store::lmdb::final_vote::end () const { - return nano::store_iterator (nullptr); + return store::iterator (nullptr); } -void nano::rocksdb::final_vote_store::for_each_par (std::function, nano::store_iterator)> const & action_a) const +void nano::store::lmdb::final_vote::for_each_par (std::function, store::iterator)> const & action_a) const { parallel_traversal ( [&action_a, this] (nano::uint512_t const & start, nano::uint512_t const & end, bool const is_last) { diff --git a/nano/store/lmdb/final_vote.hpp b/nano/store/lmdb/final_vote.hpp new file mode 100644 index 0000000000..b7c06c62fd --- /dev/null +++ b/nano/store/lmdb/final_vote.hpp @@ -0,0 +1,37 @@ +#pragma once + +#include + +#include + +namespace nano::store::lmdb +{ +class component; +} +namespace nano::store::lmdb +{ +class final_vote : public nano::store::final_vote +{ +private: + nano::store::lmdb::component & store; + +public: + explicit final_vote (nano::store::lmdb::component & store); + bool put (store::write_transaction const & transaction_a, nano::qualified_root const & root_a, nano::block_hash const & hash_a) override; + std::vector get (store::transaction const & transaction_a, nano::root const & root_a) override; + void del (store::write_transaction const & transaction_a, nano::root const & root_a) override; + size_t count (store::transaction const & transaction_a) const override; + void clear (store::write_transaction const & transaction_a, nano::root const & root_a) override; + void clear (store::write_transaction const & transaction_a) override; + store::iterator begin (store::transaction const & transaction_a, nano::qualified_root const & root_a) const override; + store::iterator begin (store::transaction const & transaction_a) const override; + store::iterator end () const override; + void for_each_par (std::function, store::iterator)> const & action_a) const override; + + /** + * Maps root to block hash for generated final votes. + * nano::qualified_root -> nano::block_hash + */ + MDB_dbi final_votes_handle{ 0 }; +}; +} // namespace nano::store::lmdb diff --git a/nano/store/lmdb/frontier.cpp b/nano/store/lmdb/frontier.cpp new file mode 100644 index 0000000000..8b694a3ef4 --- /dev/null +++ b/nano/store/lmdb/frontier.cpp @@ -0,0 +1,57 @@ +#include +#include +#include + +nano::store::lmdb::frontier::frontier (nano::store::lmdb::component & store) : + store{ store } +{ +} + +void nano::store::lmdb::frontier::put (store::write_transaction const & transaction, nano::block_hash const & hash, nano::account const & account) +{ + auto status = store.put (transaction, tables::frontiers, hash, account); + store.release_assert_success (status); +} + +nano::account nano::store::lmdb::frontier::get (store::transaction const & transaction, nano::block_hash const & hash) const +{ + store::db_val value; + auto status = store.get (transaction, tables::frontiers, hash, value); + release_assert (store.success (status) || store.not_found (status)); + nano::account result{}; + if (store.success (status)) + { + result = static_cast (value); + } + return result; +} + +void nano::store::lmdb::frontier::del (store::write_transaction const & transaction, nano::block_hash const & hash) +{ + auto status = store.del (transaction, tables::frontiers, hash); + store.release_assert_success (status); +} + +nano::store::iterator nano::store::lmdb::frontier::begin (store::transaction const & transaction) const +{ + return store.make_iterator (transaction, tables::frontiers); +} + +nano::store::iterator nano::store::lmdb::frontier::begin (store::transaction const & transaction, nano::block_hash const & hash) const +{ + return store.make_iterator (transaction, tables::frontiers, store::db_val (hash)); +} + +nano::store::iterator nano::store::lmdb::frontier::end () const +{ + return store::iterator (nullptr); +} + +void nano::store::lmdb::frontier::for_each_par (std::function, store::iterator)> const & action_a) const +{ + parallel_traversal ( + [&action_a, this] (nano::uint256_t const & start, nano::uint256_t const & end, bool const is_last) { + auto transaction (this->store.tx_begin_read ()); + action_a (transaction, this->begin (transaction, start), !is_last ? this->begin (transaction, end) : this->end ()); + }); +} diff --git a/nano/store/lmdb/frontier.hpp b/nano/store/lmdb/frontier.hpp new file mode 100644 index 0000000000..6bf71068bf --- /dev/null +++ b/nano/store/lmdb/frontier.hpp @@ -0,0 +1,34 @@ +#pragma once + +#include + +#include + +namespace nano::store::lmdb +{ +class component; +} +namespace nano::store::lmdb +{ +class frontier : public nano::store::frontier +{ +private: + nano::store::lmdb::component & store; + +public: + frontier (nano::store::lmdb::component & store); + void put (store::write_transaction const &, nano::block_hash const &, nano::account const &) override; + nano::account get (store::transaction const &, nano::block_hash const &) const override; + void del (store::write_transaction const &, nano::block_hash const &) override; + store::iterator begin (store::transaction const &) const override; + store::iterator begin (store::transaction const &, nano::block_hash const &) const override; + store::iterator end () const override; + void for_each_par (std::function, store::iterator)> const & action_a) const override; + + /** + * Maps head block to owning account + * nano::block_hash -> nano::account + */ + MDB_dbi frontiers_handle{ 0 }; +}; +} // namespace nano::store::lmdb diff --git a/nano/node/lmdb/lmdb_iterator.hpp b/nano/store/lmdb/iterator.hpp similarity index 60% rename from nano/node/lmdb/lmdb_iterator.hpp rename to nano/store/lmdb/iterator.hpp index e1e6e47b2c..9a9addf3ff 100644 --- a/nano/node/lmdb/lmdb_iterator.hpp +++ b/nano/store/lmdb/iterator.hpp @@ -1,16 +1,19 @@ #pragma once -#include +#include +#include +#include +#include #include -namespace nano +namespace nano::store::lmdb { template -class mdb_iterator : public store_iterator_impl +class iterator : public iterator_impl { public: - mdb_iterator (nano::transaction const & transaction_a, MDB_dbi db_a, MDB_val const & val_a = MDB_val{}, bool const direction_asc = true) + iterator (store::transaction const & transaction_a, MDB_dbi db_a, MDB_val const & val_a = MDB_val{}, bool const direction_asc = true) { auto status (mdb_cursor_open (tx (transaction_a), db_a, &cursor)); release_assert (status == 0); @@ -40,18 +43,18 @@ class mdb_iterator : public store_iterator_impl } } - mdb_iterator () = default; + iterator () = default; - mdb_iterator (nano::mdb_iterator && other_a) + iterator (nano::store::lmdb::iterator && other_a) { cursor = other_a.cursor; other_a.cursor = nullptr; current = other_a.current; } - mdb_iterator (nano::mdb_iterator const &) = delete; + iterator (nano::store::lmdb::iterator const &) = delete; - ~mdb_iterator () + ~iterator () { if (cursor != nullptr) { @@ -59,7 +62,7 @@ class mdb_iterator : public store_iterator_impl } } - nano::store_iterator_impl & operator++ () override + store::iterator_impl & operator++ () override { debug_assert (cursor != nullptr); auto status (mdb_cursor_get (cursor, ¤t.first.value, ¤t.second.value, MDB_NEXT)); @@ -75,7 +78,7 @@ class mdb_iterator : public store_iterator_impl return *this; } - nano::store_iterator_impl & operator-- () override + store::iterator_impl & operator-- () override { debug_assert (cursor != nullptr); auto status (mdb_cursor_get (cursor, ¤t.first.value, ¤t.second.value, MDB_PREV)); @@ -91,14 +94,14 @@ class mdb_iterator : public store_iterator_impl return *this; } - std::pair, nano::db_val> * operator-> () + std::pair, store::db_val> * operator-> () { return ¤t; } - bool operator== (nano::mdb_iterator const & base_a) const + bool operator== (nano::store::lmdb::iterator const & base_a) const { - auto const other_a (boost::polymorphic_downcast const *> (&base_a)); + auto const other_a (boost::polymorphic_downcast const *> (&base_a)); auto result (current.first.data () == other_a->current.first.data ()); debug_assert (!result || (current.first.size () == other_a->current.first.size ())); debug_assert (!result || (current.second.data () == other_a->current.second.data ())); @@ -106,9 +109,9 @@ class mdb_iterator : public store_iterator_impl return result; } - bool operator== (nano::store_iterator_impl const & base_a) const override + bool operator== (store::iterator_impl const & base_a) const override { - auto const other_a (boost::polymorphic_downcast const *> (&base_a)); + auto const other_a (boost::polymorphic_downcast const *> (&base_a)); auto result (current.first.data () == other_a->current.first.data ()); debug_assert (!result || (current.first.size () == other_a->current.first.size ())); debug_assert (!result || (current.second.data () == other_a->current.second.data ())); @@ -141,12 +144,12 @@ class mdb_iterator : public store_iterator_impl } void clear () { - current.first = nano::db_val (); - current.second = nano::db_val (); + current.first = store::db_val (); + current.second = store::db_val (); debug_assert (is_end_sentinal ()); } - nano::mdb_iterator & operator= (nano::mdb_iterator && other_a) + nano::store::lmdb::iterator & operator= (nano::store::lmdb::iterator && other_a) { if (cursor != nullptr) { @@ -159,12 +162,12 @@ class mdb_iterator : public store_iterator_impl return *this; } - nano::store_iterator_impl & operator= (nano::store_iterator_impl const &) = delete; + store::iterator_impl & operator= (store::iterator_impl const &) = delete; MDB_cursor * cursor{ nullptr }; - std::pair, nano::db_val> current; + std::pair, store::db_val> current; private: - MDB_txn * tx (nano::transaction const & transaction_a) const + MDB_txn * tx (store::transaction const & transaction_a) const { return static_cast (transaction_a.get_handle ()); } @@ -174,66 +177,66 @@ class mdb_iterator : public store_iterator_impl * Iterates the key/value pairs of two stores merged together */ template -class mdb_merge_iterator : public store_iterator_impl +class merge_iterator : public iterator_impl { public: - mdb_merge_iterator (nano::transaction const & transaction_a, MDB_dbi db1_a, MDB_dbi db2_a) : - impl1 (std::make_unique> (transaction_a, db1_a)), - impl2 (std::make_unique> (transaction_a, db2_a)) + merge_iterator (store::transaction const & transaction_a, MDB_dbi db1_a, MDB_dbi db2_a) : + impl1 (std::make_unique> (transaction_a, db1_a)), + impl2 (std::make_unique> (transaction_a, db2_a)) { } - mdb_merge_iterator () : - impl1 (std::make_unique> ()), - impl2 (std::make_unique> ()) + merge_iterator () : + impl1 (std::make_unique> ()), + impl2 (std::make_unique> ()) { } - mdb_merge_iterator (nano::transaction const & transaction_a, MDB_dbi db1_a, MDB_dbi db2_a, MDB_val const & val_a) : - impl1 (std::make_unique> (transaction_a, db1_a, val_a)), - impl2 (std::make_unique> (transaction_a, db2_a, val_a)) + merge_iterator (store::transaction const & transaction_a, MDB_dbi db1_a, MDB_dbi db2_a, MDB_val const & val_a) : + impl1 (std::make_unique> (transaction_a, db1_a, val_a)), + impl2 (std::make_unique> (transaction_a, db2_a, val_a)) { } - mdb_merge_iterator (nano::mdb_merge_iterator && other_a) + merge_iterator (merge_iterator && other_a) { impl1 = std::move (other_a.impl1); impl2 = std::move (other_a.impl2); } - mdb_merge_iterator (nano::mdb_merge_iterator const &) = delete; + merge_iterator (merge_iterator const &) = delete; - nano::store_iterator_impl & operator++ () override + store::iterator_impl & operator++ () override { ++least_iterator (); return *this; } - nano::store_iterator_impl & operator-- () override + store::iterator_impl & operator-- () override { --least_iterator (); return *this; } - std::pair, nano::db_val> * operator-> () + std::pair, store::db_val> * operator-> () { return least_iterator ().operator-> (); } - bool operator== (nano::mdb_merge_iterator const & other) const + bool operator== (merge_iterator const & other) const { return *impl1 == *other.impl1 && *impl2 == *other.impl2; } - bool operator!= (nano::mdb_merge_iterator const & base_a) const + bool operator!= (merge_iterator const & base_a) const { return !(*this == base_a); } - bool operator== (nano::store_iterator_impl const & base_a) const override + bool operator== (store::iterator_impl const & base_a) const override { - debug_assert ((dynamic_cast const *> (&base_a) != nullptr) && "Incompatible iterator comparison"); - auto & other (static_cast const &> (base_a)); + debug_assert ((dynamic_cast const *> (&base_a) != nullptr) && "Incompatible iterator comparison"); + auto & other (static_cast const &> (base_a)); return *this == other; } @@ -262,15 +265,15 @@ class mdb_merge_iterator : public store_iterator_impl value_a.second = U (); } } - nano::mdb_merge_iterator & operator= (nano::mdb_merge_iterator &&) = default; - nano::mdb_merge_iterator & operator= (nano::mdb_merge_iterator const &) = delete; + merge_iterator & operator= (merge_iterator &&) = default; + merge_iterator & operator= (merge_iterator const &) = delete; mutable bool from_first_database{ false }; private: - nano::mdb_iterator & least_iterator () const + nano::store::lmdb::iterator & least_iterator () const { - nano::mdb_iterator * result; + nano::store::lmdb::iterator * result; if (impl1->is_end_sentinal ()) { result = impl2.get (); @@ -305,7 +308,7 @@ class mdb_merge_iterator : public store_iterator_impl return *result; } - std::unique_ptr> impl1; - std::unique_ptr> impl2; + std::unique_ptr> impl1; + std::unique_ptr> impl2; }; } diff --git a/nano/store/lmdb/lmdb.cpp b/nano/store/lmdb/lmdb.cpp new file mode 100644 index 0000000000..e1385ac6da --- /dev/null +++ b/nano/store/lmdb/lmdb.cpp @@ -0,0 +1,435 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +nano::store::lmdb::component::component (nano::logger_mt & logger_a, boost::filesystem::path const & path_a, nano::ledger_constants & constants, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a, nano::lmdb_config const & lmdb_config_a, bool backup_before_upgrade_a) : + // clang-format off + nano::store::component{ + block_store, + frontier_store, + account_store, + pending_store, + online_weight_store, + pruned_store, + peer_store, + confirmation_height_store, + final_vote_store, + version_store + }, + // clang-format on + block_store{ *this }, + frontier_store{ *this }, + account_store{ *this }, + pending_store{ *this }, + online_weight_store{ *this }, + pruned_store{ *this }, + peer_store{ *this }, + confirmation_height_store{ *this }, + final_vote_store{ *this }, + version_store{ *this }, + logger (logger_a), + env (error, path_a, nano::store::lmdb::env::options::make ().set_config (lmdb_config_a).set_use_no_mem_init (true)), + mdb_txn_tracker (logger_a, txn_tracking_config_a, block_processor_batch_max_time_a), + txn_tracking_enabled (txn_tracking_config_a.enable) +{ + if (!error) + { + debug_assert (path_a.filename () == "data.ldb"); + + auto is_fully_upgraded (false); + auto is_fresh_db (false); + { + auto transaction (tx_begin_read ()); + auto err = mdb_dbi_open (env.tx (transaction), "meta", 0, &version_store.meta_handle); + is_fresh_db = err != MDB_SUCCESS; + if (err == MDB_SUCCESS) + { + is_fully_upgraded = (version.get (transaction) == version_current); + mdb_dbi_close (env, version_store.meta_handle); + } + } + + // Only open a write lock when upgrades are needed. This is because CLI commands + // open inactive nodes which can otherwise be locked here if there is a long write + // (can be a few minutes with the --fast_bootstrap flag for instance) + if (!is_fully_upgraded) + { + if (!is_fresh_db) + { + logger.always_log ("Upgrade in progress..."); + if (backup_before_upgrade_a) + { + create_backup_file (env, path_a, logger_a); + } + } + auto needs_vacuuming = false; + { + auto transaction (tx_begin_write ()); + open_databases (error, transaction, MDB_CREATE); + if (!error) + { + error |= do_upgrades (transaction, constants, needs_vacuuming); + } + } + + if (needs_vacuuming) + { + logger.always_log ("Preparing vacuum..."); + auto vacuum_success = vacuum_after_upgrade (path_a, lmdb_config_a); + logger.always_log (vacuum_success ? "Vacuum succeeded." : "Failed to vacuum. (Optional) Ensure enough disk space is available for a copy of the database and try to vacuum after shutting down the node"); + } + } + else + { + auto transaction (tx_begin_read ()); + open_databases (error, transaction, 0); + } + } +} + +bool nano::store::lmdb::component::vacuum_after_upgrade (boost::filesystem::path const & path_a, nano::lmdb_config const & lmdb_config_a) +{ + // Vacuum the database. This is not a required step and may actually fail if there isn't enough storage space. + auto vacuum_path = path_a.parent_path () / "vacuumed.ldb"; + + auto vacuum_success = copy_db (vacuum_path); + if (vacuum_success) + { + // Need to close the database to release the file handle + mdb_env_sync (env.environment, true); + mdb_env_close (env.environment); + env.environment = nullptr; + + // Replace the ledger file with the vacuumed one + boost::filesystem::rename (vacuum_path, path_a); + + // Set up the environment again + auto options = nano::store::lmdb::env::options::make () + .set_config (lmdb_config_a) + .set_use_no_mem_init (true); + env.init (error, path_a, options); + if (!error) + { + auto transaction (tx_begin_read ()); + open_databases (error, transaction, 0); + } + } + else + { + // The vacuum file can be in an inconsistent state if there wasn't enough space to create it + boost::filesystem::remove (vacuum_path); + } + return vacuum_success; +} + +void nano::store::lmdb::component::serialize_mdb_tracker (boost::property_tree::ptree & json, std::chrono::milliseconds min_read_time, std::chrono::milliseconds min_write_time) +{ + mdb_txn_tracker.serialize_json (json, min_read_time, min_write_time); +} + +void nano::store::lmdb::component::serialize_memory_stats (boost::property_tree::ptree & json) +{ + MDB_stat stats; + auto status (mdb_env_stat (env.environment, &stats)); + release_assert (status == 0); + json.put ("branch_pages", stats.ms_branch_pages); + json.put ("depth", stats.ms_depth); + json.put ("entries", stats.ms_entries); + json.put ("leaf_pages", stats.ms_leaf_pages); + json.put ("overflow_pages", stats.ms_overflow_pages); + json.put ("page_size", stats.ms_psize); +} + +nano::store::write_transaction nano::store::lmdb::component::tx_begin_write (std::vector const &, std::vector const &) +{ + return env.tx_begin_write (create_txn_callbacks ()); +} + +nano::store::read_transaction nano::store::lmdb::component::tx_begin_read () const +{ + return env.tx_begin_read (create_txn_callbacks ()); +} + +std::string nano::store::lmdb::component::vendor_get () const +{ + return boost::str (boost::format ("LMDB %1%.%2%.%3%") % MDB_VERSION_MAJOR % MDB_VERSION_MINOR % MDB_VERSION_PATCH); +} + +nano::store::lmdb::txn_callbacks nano::store::lmdb::component::create_txn_callbacks () const +{ + nano::store::lmdb::txn_callbacks mdb_txn_callbacks; + if (txn_tracking_enabled) + { + mdb_txn_callbacks.txn_start = ([&mdb_txn_tracker = mdb_txn_tracker] (store::transaction_impl const * transaction_impl) { + mdb_txn_tracker.add (transaction_impl); + }); + mdb_txn_callbacks.txn_end = ([&mdb_txn_tracker = mdb_txn_tracker] (store::transaction_impl const * transaction_impl) { + mdb_txn_tracker.erase (transaction_impl); + }); + } + return mdb_txn_callbacks; +} + +void nano::store::lmdb::component::open_databases (bool & error_a, store::transaction const & transaction_a, unsigned flags) +{ + error_a |= mdb_dbi_open (env.tx (transaction_a), "frontiers", flags, &frontier_store.frontiers_handle) != 0; + error_a |= mdb_dbi_open (env.tx (transaction_a), "online_weight", flags, &online_weight_store.online_weight_handle) != 0; + error_a |= mdb_dbi_open (env.tx (transaction_a), "meta", flags, &version_store.meta_handle) != 0; + error_a |= mdb_dbi_open (env.tx (transaction_a), "peers", flags, &peer_store.peers_handle) != 0; + error_a |= mdb_dbi_open (env.tx (transaction_a), "pruned", flags, &pruned_store.pruned_handle) != 0; + error_a |= mdb_dbi_open (env.tx (transaction_a), "confirmation_height", flags, &confirmation_height_store.confirmation_height_handle) != 0; + error_a |= mdb_dbi_open (env.tx (transaction_a), "accounts", flags, &account_store.accounts_v0_handle) != 0; + account_store.accounts_handle = account_store.accounts_v0_handle; + error_a |= mdb_dbi_open (env.tx (transaction_a), "pending", flags, &pending_store.pending_v0_handle) != 0; + pending_store.pending_handle = pending_store.pending_v0_handle; + error_a |= mdb_dbi_open (env.tx (transaction_a), "final_votes", flags, &final_vote_store.final_votes_handle) != 0; + error_a |= mdb_dbi_open (env.tx (transaction_a), "blocks", MDB_CREATE, &block_store.blocks_handle) != 0; +} + +bool nano::store::lmdb::component::do_upgrades (store::write_transaction & transaction_a, nano::ledger_constants & constants, bool & needs_vacuuming) +{ + auto error (false); + auto version_l = version.get (transaction_a); + if (version_l < version_minimum) + { + logger.always_log (boost::str (boost::format ("The version of the ledger (%1%) is lower than the minimum (%2%) which is supported for upgrades. Either upgrade to a v19, v20 or v21 node first or delete the ledger.") % version_l % version_minimum)); + return true; + } + switch (version_l) + { + case 21: + upgrade_v21_to_v22 (transaction_a); + [[fallthrough]]; + case 22: + break; + default: + logger.always_log (boost::str (boost::format ("The version of the ledger (%1%) is too high for this node") % version_l)); + error = true; + break; + } + return error; +} + +void nano::store::lmdb::component::upgrade_v21_to_v22 (store::write_transaction const & transaction_a) +{ + logger.always_log ("Preparing v21 to v22 database upgrade..."); + MDB_dbi unchecked_handle{ 0 }; + release_assert (!mdb_dbi_open (env.tx (transaction_a), "unchecked", MDB_CREATE, &unchecked_handle)); + release_assert (!mdb_drop (env.tx (transaction_a), unchecked_handle, 1)); // del = 1, to delete it from the environment and close the DB handle. + version.put (transaction_a, 22); + logger.always_log ("Finished removing unchecked table"); +} + +/** Takes a filepath, appends '_backup_' to the end (but before any extension) and saves that file in the same directory */ +void nano::store::lmdb::component::create_backup_file (nano::store::lmdb::env & env_a, boost::filesystem::path const & filepath_a, nano::logger_mt & logger_a) +{ + auto extension = filepath_a.extension (); + auto filename_without_extension = filepath_a.filename ().replace_extension (""); + auto orig_filepath = filepath_a; + auto & backup_path = orig_filepath.remove_filename (); + auto backup_filename = filename_without_extension; + backup_filename += "_backup_"; + backup_filename += std::to_string (std::chrono::system_clock::now ().time_since_epoch ().count ()); + backup_filename += extension; + auto backup_filepath = backup_path / backup_filename; + auto start_message (boost::str (boost::format ("Performing %1% backup before database upgrade...") % filepath_a.filename ())); + logger_a.always_log (start_message); + std::cout << start_message << std::endl; + auto error (mdb_env_copy (env_a, backup_filepath.string ().c_str ())); + if (error) + { + auto error_message (boost::str (boost::format ("%1% backup failed") % filepath_a.filename ())); + logger_a.always_log (error_message); + std::cerr << error_message << std::endl; + std::exit (1); + } + else + { + auto success_message (boost::str (boost::format ("Backup created: %1%") % backup_filename)); + logger_a.always_log (success_message); + std::cout << success_message << std::endl; + } +} + +bool nano::store::lmdb::component::exists (store::transaction const & transaction_a, tables table_a, nano::store::lmdb::db_val const & key_a) const +{ + nano::store::lmdb::db_val junk; + auto status = get (transaction_a, table_a, key_a, junk); + release_assert (status == MDB_SUCCESS || status == MDB_NOTFOUND); + return (status == MDB_SUCCESS); +} + +int nano::store::lmdb::component::get (store::transaction const & transaction_a, tables table_a, nano::store::lmdb::db_val const & key_a, nano::store::lmdb::db_val & value_a) const +{ + return mdb_get (env.tx (transaction_a), table_to_dbi (table_a), key_a, value_a); +} + +int nano::store::lmdb::component::put (store::write_transaction const & transaction_a, tables table_a, nano::store::lmdb::db_val const & key_a, nano::store::lmdb::db_val const & value_a) const +{ + return (mdb_put (env.tx (transaction_a), table_to_dbi (table_a), key_a, value_a, 0)); +} + +int nano::store::lmdb::component::del (store::write_transaction const & transaction_a, tables table_a, nano::store::lmdb::db_val const & key_a) const +{ + return (mdb_del (env.tx (transaction_a), table_to_dbi (table_a), key_a, nullptr)); +} + +int nano::store::lmdb::component::drop (store::write_transaction const & transaction_a, tables table_a) +{ + return clear (transaction_a, table_to_dbi (table_a)); +} + +int nano::store::lmdb::component::clear (store::write_transaction const & transaction_a, MDB_dbi handle_a) +{ + return mdb_drop (env.tx (transaction_a), handle_a, 0); +} + +uint64_t nano::store::lmdb::component::count (store::transaction const & transaction_a, tables table_a) const +{ + return count (transaction_a, table_to_dbi (table_a)); +} + +uint64_t nano::store::lmdb::component::count (store::transaction const & transaction_a, MDB_dbi db_a) const +{ + MDB_stat stats; + auto status (mdb_stat (env.tx (transaction_a), db_a, &stats)); + release_assert_success (status); + return (stats.ms_entries); +} + +MDB_dbi nano::store::lmdb::component::table_to_dbi (tables table_a) const +{ + switch (table_a) + { + case tables::frontiers: + return frontier_store.frontiers_handle; + case tables::accounts: + return account_store.accounts_handle; + case tables::blocks: + return block_store.blocks_handle; + case tables::pending: + return pending_store.pending_handle; + case tables::online_weight: + return online_weight_store.online_weight_handle; + case tables::meta: + return version_store.meta_handle; + case tables::peers: + return peer_store.peers_handle; + case tables::pruned: + return pruned_store.pruned_handle; + case tables::confirmation_height: + return confirmation_height_store.confirmation_height_handle; + case tables::final_votes: + return final_vote_store.final_votes_handle; + default: + release_assert (false); + return peer_store.peers_handle; + } +} + +bool nano::store::lmdb::component::not_found (int status) const +{ + return (status_code_not_found () == status); +} + +bool nano::store::lmdb::component::success (int status) const +{ + return (MDB_SUCCESS == status); +} + +int nano::store::lmdb::component::status_code_not_found () const +{ + return MDB_NOTFOUND; +} + +std::string nano::store::lmdb::component::error_string (int status) const +{ + return mdb_strerror (status); +} + +bool nano::store::lmdb::component::copy_db (boost::filesystem::path const & destination_file) +{ + return !mdb_env_copy2 (env.environment, destination_file.string ().c_str (), MDB_CP_COMPACT); +} + +void nano::store::lmdb::component::rebuild_db (store::write_transaction const & transaction_a) +{ + // Tables with uint256_union key + std::vector tables = { account_store.accounts_handle, block_store.blocks_handle, pruned_store.pruned_handle, confirmation_height_store.confirmation_height_handle }; + for (auto const & table : tables) + { + MDB_dbi temp; + mdb_dbi_open (env.tx (transaction_a), "temp_table", MDB_CREATE, &temp); + // Copy all values to temporary table + for (auto i (store::iterator (std::make_unique> (transaction_a, table))), n (store::iterator (nullptr)); i != n; ++i) + { + auto s = mdb_put (env.tx (transaction_a), temp, nano::store::lmdb::db_val (i->first), i->second, MDB_APPEND); + release_assert_success (s); + } + release_assert (count (transaction_a, table) == count (transaction_a, temp)); + // Clear existing table + mdb_drop (env.tx (transaction_a), table, 0); + // Put values from copy + for (auto i (store::iterator (std::make_unique> (transaction_a, temp))), n (store::iterator (nullptr)); i != n; ++i) + { + auto s = mdb_put (env.tx (transaction_a), table, nano::store::lmdb::db_val (i->first), i->second, MDB_APPEND); + release_assert_success (s); + } + release_assert (count (transaction_a, table) == count (transaction_a, temp)); + // Remove temporary table + mdb_drop (env.tx (transaction_a), temp, 1); + } + // Pending table + { + MDB_dbi temp; + mdb_dbi_open (env.tx (transaction_a), "temp_table", MDB_CREATE, &temp); + // Copy all values to temporary table + for (auto i (store::iterator (std::make_unique> (transaction_a, pending_store.pending_handle))), n (store::iterator (nullptr)); i != n; ++i) + { + auto s = mdb_put (env.tx (transaction_a), temp, nano::store::lmdb::db_val (i->first), nano::store::lmdb::db_val (i->second), MDB_APPEND); + release_assert_success (s); + } + release_assert (count (transaction_a, pending_store.pending_handle) == count (transaction_a, temp)); + mdb_drop (env.tx (transaction_a), pending_store.pending_handle, 0); + // Put values from copy + for (auto i (store::iterator (std::make_unique> (transaction_a, temp))), n (store::iterator (nullptr)); i != n; ++i) + { + auto s = mdb_put (env.tx (transaction_a), pending_store.pending_handle, nano::store::lmdb::db_val (i->first), nano::store::lmdb::db_val (i->second), MDB_APPEND); + release_assert_success (s); + } + release_assert (count (transaction_a, pending_store.pending_handle) == count (transaction_a, temp)); + mdb_drop (env.tx (transaction_a), temp, 1); + } +} + +bool nano::store::lmdb::component::init_error () const +{ + return error; +} + +nano::store::lmdb::component::upgrade_counters::upgrade_counters (uint64_t count_before_v0, uint64_t count_before_v1) : + before_v0 (count_before_v0), + before_v1 (count_before_v1) +{ +} + +bool nano::store::lmdb::component::upgrade_counters::are_equal () const +{ + return (before_v0 == after_v0) && (before_v1 == after_v1); +} + +unsigned nano::store::lmdb::component::max_block_write_batch_num () const +{ + return std::numeric_limits::max (); +} diff --git a/nano/store/lmdb/lmdb.hpp b/nano/store/lmdb/lmdb.hpp new file mode 100644 index 0000000000..cd741d5c9e --- /dev/null +++ b/nano/store/lmdb/lmdb.hpp @@ -0,0 +1,167 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +namespace boost +{ +namespace filesystem +{ + class path; +} +} + +namespace nano +{ +class logging_mt; + +} + +namespace nano::store::lmdb +{ +/** + * mdb implementation of the block store + */ +class component : public nano::store::component +{ +private: + nano::store::lmdb::account account_store; + nano::store::lmdb::block block_store; + nano::store::lmdb::confirmation_height confirmation_height_store; + nano::store::lmdb::final_vote final_vote_store; + nano::store::lmdb::frontier frontier_store; + nano::store::lmdb::online_weight online_weight_store; + nano::store::lmdb::peer peer_store; + nano::store::lmdb::pending pending_store; + nano::store::lmdb::pruned pruned_store; + nano::store::lmdb::version version_store; + + friend class nano::store::lmdb::account; + friend class nano::store::lmdb::block; + friend class nano::store::lmdb::confirmation_height; + friend class nano::store::lmdb::final_vote; + friend class nano::store::lmdb::frontier; + friend class nano::store::lmdb::online_weight; + friend class nano::store::lmdb::peer; + friend class nano::store::lmdb::pending; + friend class nano::store::lmdb::pruned; + friend class nano::store::lmdb::version; + +public: + component (nano::logger_mt &, boost::filesystem::path const &, nano::ledger_constants & constants, nano::txn_tracking_config const & txn_tracking_config_a = nano::txn_tracking_config{}, std::chrono::milliseconds block_processor_batch_max_time_a = std::chrono::milliseconds (5000), nano::lmdb_config const & lmdb_config_a = nano::lmdb_config{}, bool backup_before_upgrade = false); + store::write_transaction tx_begin_write (std::vector const & tables_requiring_lock = {}, std::vector const & tables_no_lock = {}) override; + store::read_transaction tx_begin_read () const override; + + std::string vendor_get () const override; + + void serialize_mdb_tracker (boost::property_tree::ptree &, std::chrono::milliseconds, std::chrono::milliseconds) override; + + static void create_backup_file (nano::store::lmdb::env &, boost::filesystem::path const &, nano::logger_mt &); + + void serialize_memory_stats (boost::property_tree::ptree &) override; + + unsigned max_block_write_batch_num () const override; + +private: + nano::logger_mt & logger; + bool error{ false }; + +public: + nano::store::lmdb::env env; + + bool exists (store::transaction const & transaction_a, tables table_a, nano::store::lmdb::db_val const & key_a) const; + + int get (store::transaction const & transaction_a, tables table_a, nano::store::lmdb::db_val const & key_a, nano::store::lmdb::db_val & value_a) const; + int put (store::write_transaction const & transaction_a, tables table_a, nano::store::lmdb::db_val const & key_a, nano::store::lmdb::db_val const & value_a) const; + int del (store::write_transaction const & transaction_a, tables table_a, nano::store::lmdb::db_val const & key_a) const; + + bool copy_db (boost::filesystem::path const & destination_file) override; + void rebuild_db (store::write_transaction const & transaction_a) override; + + template + store::iterator make_iterator (store::transaction const & transaction_a, tables table_a, bool const direction_asc = true) const + { + return store::iterator (std::make_unique> (transaction_a, table_to_dbi (table_a), nano::store::lmdb::db_val{}, direction_asc)); + } + + template + store::iterator make_iterator (store::transaction const & transaction_a, tables table_a, nano::store::lmdb::db_val const & key) const + { + return store::iterator (std::make_unique> (transaction_a, table_to_dbi (table_a), key)); + } + + bool init_error () const override; + + uint64_t count (store::transaction const &, MDB_dbi) const; + std::string error_string (int status) const override; + +private: + bool do_upgrades (store::write_transaction &, nano::ledger_constants & constants, bool &); + void upgrade_v21_to_v22 (store::write_transaction const &); + + void open_databases (bool &, store::transaction const &, unsigned); + + int drop (store::write_transaction const & transaction_a, tables table_a) override; + int clear (store::write_transaction const & transaction_a, MDB_dbi handle_a); + + bool not_found (int status) const override; + bool success (int status) const override; + void release_assert_success (int const status) const + { + if (!success (status)) + { + release_assert (false, error_string (status)); + } + } + int status_code_not_found () const override; + + MDB_dbi table_to_dbi (tables table_a) const; + + mutable nano::mdb_txn_tracker mdb_txn_tracker; + nano::store::lmdb::txn_callbacks create_txn_callbacks () const; + bool txn_tracking_enabled; + + uint64_t count (store::transaction const & transaction_a, tables table_a) const override; + + bool vacuum_after_upgrade (boost::filesystem::path const & path_a, nano::lmdb_config const & lmdb_config_a); + + class upgrade_counters + { + public: + upgrade_counters (uint64_t count_before_v0, uint64_t count_before_v1); + bool are_equal () const; + + uint64_t before_v0; + uint64_t before_v1; + uint64_t after_v0{ 0 }; + uint64_t after_v1{ 0 }; + }; + + friend class mdb_block_store_supported_version_upgrades_Test; + friend class mdb_block_store_upgrade_v21_v22_Test; + friend class block_store_DISABLED_change_dupsort_Test; +}; +} // namespace nano::store::lmdb diff --git a/nano/node/lmdb/lmdb_env.cpp b/nano/store/lmdb/lmdb_env.cpp similarity index 75% rename from nano/node/lmdb/lmdb_env.cpp rename to nano/store/lmdb/lmdb_env.cpp index 394b13ea6b..b34289e7f1 100644 --- a/nano/node/lmdb/lmdb_env.cpp +++ b/nano/store/lmdb/lmdb_env.cpp @@ -1,13 +1,13 @@ -#include +#include #include -nano::mdb_env::mdb_env (bool & error_a, boost::filesystem::path const & path_a, nano::mdb_env::options options_a) +nano::store::lmdb::env::env (bool & error_a, boost::filesystem::path const & path_a, nano::store::lmdb::env::options options_a) { init (error_a, path_a, options_a); } -void nano::mdb_env::init (bool & error_a, boost::filesystem::path const & path_a, nano::mdb_env::options options_a) +void nano::store::lmdb::env::init (bool & error_a, boost::filesystem::path const & path_a, nano::store::lmdb::env::options options_a) { boost::system::error_code error_mkdir, error_chmod; if (path_a.has_parent_path ()) @@ -78,7 +78,7 @@ void nano::mdb_env::init (bool & error_a, boost::filesystem::path const & path_a } } -nano::mdb_env::~mdb_env () +nano::store::lmdb::env::~env () { if (environment != nullptr) { @@ -88,22 +88,22 @@ nano::mdb_env::~mdb_env () } } -nano::mdb_env::operator MDB_env * () const +nano::store::lmdb::env::operator MDB_env * () const { return environment; } -nano::read_transaction nano::mdb_env::tx_begin_read (mdb_txn_callbacks mdb_txn_callbacks) const +nano::store::read_transaction nano::store::lmdb::env::tx_begin_read (store::lmdb::txn_callbacks mdb_txn_callbacks) const { - return nano::read_transaction{ std::make_unique (*this, mdb_txn_callbacks) }; + return store::read_transaction{ std::make_unique (*this, mdb_txn_callbacks) }; } -nano::write_transaction nano::mdb_env::tx_begin_write (mdb_txn_callbacks mdb_txn_callbacks) const +nano::store::write_transaction nano::store::lmdb::env::tx_begin_write (store::lmdb::txn_callbacks mdb_txn_callbacks) const { - return nano::write_transaction{ std::make_unique (*this, mdb_txn_callbacks) }; + return store::write_transaction{ std::make_unique (*this, mdb_txn_callbacks) }; } -MDB_txn * nano::mdb_env::tx (nano::transaction const & transaction_a) const +MDB_txn * nano::store::lmdb::env::tx (store::transaction const & transaction_a) const { return static_cast (transaction_a.get_handle ()); } diff --git a/nano/node/lmdb/lmdb_env.hpp b/nano/store/lmdb/lmdb_env.hpp similarity index 60% rename from nano/node/lmdb/lmdb_env.hpp rename to nano/store/lmdb/lmdb_env.hpp index 6fa3a082d0..21634509db 100644 --- a/nano/node/lmdb/lmdb_env.hpp +++ b/nano/store/lmdb/lmdb_env.hpp @@ -1,21 +1,21 @@ #pragma once #include -#include -#include +#include +#include -namespace nano +namespace nano::store::lmdb { /** * RAII wrapper for MDB_env */ -class mdb_env final +class env final { public: /** Environment options, most of which originates from the config file. */ class options final { - friend class mdb_env; + friend class env; public: static options make () @@ -54,13 +54,13 @@ class mdb_env final nano::lmdb_config config; }; - mdb_env (bool &, boost::filesystem::path const &, nano::mdb_env::options options_a = nano::mdb_env::options::make ()); - void init (bool &, boost::filesystem::path const &, nano::mdb_env::options options_a = nano::mdb_env::options::make ()); - ~mdb_env (); + env (bool &, boost::filesystem::path const &, env::options options_a = env::options::make ()); + void init (bool &, boost::filesystem::path const &, env::options options_a = env::options::make ()); + ~env (); operator MDB_env * () const; - nano::read_transaction tx_begin_read (mdb_txn_callbacks txn_callbacks = mdb_txn_callbacks{}) const; - nano::write_transaction tx_begin_write (mdb_txn_callbacks txn_callbacks = mdb_txn_callbacks{}) const; - MDB_txn * tx (nano::transaction const & transaction_a) const; + store::read_transaction tx_begin_read (txn_callbacks callbacks = txn_callbacks{}) const; + store::write_transaction tx_begin_write (txn_callbacks callbacks = txn_callbacks{}) const; + MDB_txn * tx (store::transaction const & transaction_a) const; MDB_env * environment; }; -} +} // namespace nano::store::lmdb diff --git a/nano/store/lmdb/online_weight.cpp b/nano/store/lmdb/online_weight.cpp new file mode 100644 index 0000000000..647aaf34da --- /dev/null +++ b/nano/store/lmdb/online_weight.cpp @@ -0,0 +1,45 @@ +#include +#include + +nano::store::lmdb::online_weight::online_weight (nano::store::lmdb::component & store_a) : + store{ store_a } +{ +} + +void nano::store::lmdb::online_weight::put (store::write_transaction const & transaction, uint64_t time, nano::amount const & amount) +{ + auto status = store.put (transaction, tables::online_weight, time, amount); + store.release_assert_success (status); +} + +void nano::store::lmdb::online_weight::del (store::write_transaction const & transaction, uint64_t time) +{ + auto status = store.del (transaction, tables::online_weight, time); + store.release_assert_success (status); +} + +nano::store::iterator nano::store::lmdb::online_weight::begin (store::transaction const & transaction) const +{ + return store.make_iterator (transaction, tables::online_weight); +} + +nano::store::iterator nano::store::lmdb::online_weight::rbegin (store::transaction const & transaction) const +{ + return store.make_iterator (transaction, tables::online_weight, false); +} + +nano::store::iterator nano::store::lmdb::online_weight::end () const +{ + return store::iterator (nullptr); +} + +size_t nano::store::lmdb::online_weight::count (store::transaction const & transaction) const +{ + return store.count (transaction, tables::online_weight); +} + +void nano::store::lmdb::online_weight::clear (store::write_transaction const & transaction) +{ + auto status = store.drop (transaction, tables::online_weight); + store.release_assert_success (status); +} diff --git a/nano/store/lmdb/online_weight.hpp b/nano/store/lmdb/online_weight.hpp new file mode 100644 index 0000000000..df70dfb9d0 --- /dev/null +++ b/nano/store/lmdb/online_weight.hpp @@ -0,0 +1,30 @@ +#pragma once + +#include + +#include + +namespace nano::store::lmdb +{ +class online_weight : public nano::store::online_weight +{ +private: + nano::store::lmdb::component & store; + +public: + explicit online_weight (nano::store::lmdb::component & store_a); + void put (store::write_transaction const & transaction_a, uint64_t time_a, nano::amount const & amount_a) override; + void del (store::write_transaction const & transaction_a, uint64_t time_a) override; + store::iterator begin (store::transaction const & transaction_a) const override; + store::iterator rbegin (store::transaction const & transaction_a) const override; + store::iterator end () const override; + size_t count (store::transaction const & transaction_a) const override; + void clear (store::write_transaction const & transaction_a) override; + + /** + * Samples of online vote weight + * uint64_t -> nano::amount + */ + MDB_dbi online_weight_handle{ 0 }; +}; +} // namespace nano::store::lmdb diff --git a/nano/store/lmdb/peer.cpp b/nano/store/lmdb/peer.cpp new file mode 100644 index 0000000000..cb3de165c5 --- /dev/null +++ b/nano/store/lmdb/peer.cpp @@ -0,0 +1,43 @@ +#include +#include + +nano::store::lmdb::peer::peer (nano::store::lmdb::component & store) : + store{ store } {}; + +void nano::store::lmdb::peer::put (store::write_transaction const & transaction, nano::endpoint_key const & endpoint) +{ + auto status = store.put (transaction, tables::peers, endpoint, nullptr); + store.release_assert_success (status); +} + +void nano::store::lmdb::peer::del (store::write_transaction const & transaction, nano::endpoint_key const & endpoint) +{ + auto status = store.del (transaction, tables::peers, endpoint); + store.release_assert_success (status); +} + +bool nano::store::lmdb::peer::exists (store::transaction const & transaction, nano::endpoint_key const & endpoint) const +{ + return store.exists (transaction, tables::peers, endpoint); +} + +size_t nano::store::lmdb::peer::count (store::transaction const & transaction) const +{ + return store.count (transaction, tables::peers); +} + +void nano::store::lmdb::peer::clear (store::write_transaction const & transaction) +{ + auto status = store.drop (transaction, tables::peers); + store.release_assert_success (status); +} + +nano::store::iterator nano::store::lmdb::peer::begin (store::transaction const & transaction) const +{ + return store.make_iterator (transaction, tables::peers); +} + +nano::store::iterator nano::store::lmdb::peer::end () const +{ + return store::iterator (nullptr); +} diff --git a/nano/store/lmdb/peer.hpp b/nano/store/lmdb/peer.hpp new file mode 100644 index 0000000000..7519908eee --- /dev/null +++ b/nano/store/lmdb/peer.hpp @@ -0,0 +1,30 @@ +#pragma once + +#include + +#include + +namespace nano::store::lmdb +{ +class peer : public nano::store::peer +{ +private: + nano::store::lmdb::component & store; + +public: + explicit peer (nano::store::lmdb::component & store_a); + void put (store::write_transaction const & transaction_a, nano::endpoint_key const & endpoint_a) override; + void del (store::write_transaction const & transaction_a, nano::endpoint_key const & endpoint_a) override; + bool exists (store::transaction const & transaction_a, nano::endpoint_key const & endpoint_a) const override; + size_t count (store::transaction const & transaction_a) const override; + void clear (store::write_transaction const & transaction_a) override; + store::iterator begin (store::transaction const & transaction_a) const override; + store::iterator end () const override; + + /* + * Endpoints for peers + * nano::endpoint_key -> no_value + */ + MDB_dbi peers_handle{ 0 }; +}; +} // namespace nano::store::lmdb diff --git a/nano/node/lmdb/pending_store.cpp b/nano/store/lmdb/pending.cpp similarity index 53% rename from nano/node/lmdb/pending_store.cpp rename to nano/store/lmdb/pending.cpp index dc4e77691b..961b201829 100644 --- a/nano/node/lmdb/pending_store.cpp +++ b/nano/store/lmdb/pending.cpp @@ -1,25 +1,25 @@ -#include -#include #include +#include +#include -nano::lmdb::pending_store::pending_store (nano::lmdb::store & store) : +nano::store::lmdb::pending::pending (nano::store::lmdb::component & store) : store{ store } {}; -void nano::lmdb::pending_store::put (nano::write_transaction const & transaction, nano::pending_key const & key, nano::pending_info const & pending) +void nano::store::lmdb::pending::put (store::write_transaction const & transaction, nano::pending_key const & key, nano::pending_info const & pending) { auto status = store.put (transaction, tables::pending, key, pending); store.release_assert_success (status); } -void nano::lmdb::pending_store::del (nano::write_transaction const & transaction, nano::pending_key const & key) +void nano::store::lmdb::pending::del (store::write_transaction const & transaction, nano::pending_key const & key) { auto status = store.del (transaction, tables::pending, key); store.release_assert_success (status); } -bool nano::lmdb::pending_store::get (nano::transaction const & transaction, nano::pending_key const & key, nano::pending_info & pending_a) +bool nano::store::lmdb::pending::get (store::transaction const & transaction, nano::pending_key const & key, nano::pending_info & pending_a) { - nano::mdb_val value; + nano::store::lmdb::db_val value; auto status1 = store.get (transaction, tables::pending, key, value); release_assert (store.success (status1) || store.not_found (status1)); bool result (true); @@ -31,34 +31,34 @@ bool nano::lmdb::pending_store::get (nano::transaction const & transaction, nano return result; } -bool nano::lmdb::pending_store::exists (nano::transaction const & transaction_a, nano::pending_key const & key_a) +bool nano::store::lmdb::pending::exists (store::transaction const & transaction_a, nano::pending_key const & key_a) { auto iterator (begin (transaction_a, key_a)); return iterator != end () && nano::pending_key (iterator->first) == key_a; } -bool nano::lmdb::pending_store::any (nano::transaction const & transaction_a, nano::account const & account_a) +bool nano::store::lmdb::pending::any (store::transaction const & transaction_a, nano::account const & account_a) { auto iterator (begin (transaction_a, nano::pending_key (account_a, 0))); return iterator != end () && nano::pending_key (iterator->first).account == account_a; } -nano::store_iterator nano::lmdb::pending_store::begin (nano::transaction const & transaction_a, nano::pending_key const & key_a) const +nano::store::iterator nano::store::lmdb::pending::begin (store::transaction const & transaction_a, nano::pending_key const & key_a) const { return store.make_iterator (transaction_a, tables::pending, key_a); } -nano::store_iterator nano::lmdb::pending_store::begin (nano::transaction const & transaction_a) const +nano::store::iterator nano::store::lmdb::pending::begin (store::transaction const & transaction_a) const { return store.make_iterator (transaction_a, tables::pending); } -nano::store_iterator nano::lmdb::pending_store::end () const +nano::store::iterator nano::store::lmdb::pending::end () const { - return nano::store_iterator (nullptr); + return store::iterator (nullptr); } -void nano::lmdb::pending_store::for_each_par (std::function, nano::store_iterator)> const & action_a) const +void nano::store::lmdb::pending::for_each_par (std::function, store::iterator)> const & action_a) const { parallel_traversal ( [&action_a, this] (nano::uint512_t const & start, nano::uint512_t const & end, bool const is_last) { diff --git a/nano/store/lmdb/pending.hpp b/nano/store/lmdb/pending.hpp new file mode 100644 index 0000000000..7b39a9132c --- /dev/null +++ b/nano/store/lmdb/pending.hpp @@ -0,0 +1,48 @@ +#pragma once + +#include + +#include + +namespace nano::store::lmdb +{ +class component; +} +namespace nano::store::lmdb +{ +class pending : public nano::store::pending +{ +private: + nano::store::lmdb::component & store; + +public: + explicit pending (nano::store::lmdb::component & store_a); + void put (store::write_transaction const & transaction_a, nano::pending_key const & key_a, nano::pending_info const & pending_info_a) override; + void del (store::write_transaction const & transaction_a, nano::pending_key const & key_a) override; + bool get (store::transaction const & transaction_a, nano::pending_key const & key_a, nano::pending_info & pending_a) override; + bool exists (store::transaction const & transaction_a, nano::pending_key const & key_a) override; + bool any (store::transaction const & transaction_a, nano::account const & account_a) override; + store::iterator begin (store::transaction const & transaction_a, nano::pending_key const & key_a) const override; + store::iterator begin (store::transaction const & transaction_a) const override; + store::iterator end () const override; + void for_each_par (std::function, store::iterator)> const & action_a) const override; + + /** + * Maps min_version 0 (destination account, pending block) to (source account, amount). (Removed) + * nano::account, nano::block_hash -> nano::account, nano::amount + */ + MDB_dbi pending_v0_handle{ 0 }; + + /** + * Maps min_version 1 (destination account, pending block) to (source account, amount). (Removed) + * nano::account, nano::block_hash -> nano::account, nano::amount + */ + MDB_dbi pending_v1_handle{ 0 }; + + /** + * Maps (destination account, pending block) to (source account, amount, version). (Removed) + * nano::account, nano::block_hash -> nano::account, nano::amount, nano::epoch + */ + MDB_dbi pending_handle{ 0 }; +}; +} // namespace nano::store::lmdb diff --git a/nano/store/lmdb/pruned.cpp b/nano/store/lmdb/pruned.cpp new file mode 100644 index 0000000000..2be19c24f5 --- /dev/null +++ b/nano/store/lmdb/pruned.cpp @@ -0,0 +1,70 @@ +#include +#include +#include + +nano::store::lmdb::pruned::pruned (nano::store::lmdb::component & store_a) : + store{ store_a } {}; + +void nano::store::lmdb::pruned::put (store::write_transaction const & transaction_a, nano::block_hash const & hash_a) +{ + auto status = store.put (transaction_a, tables::pruned, hash_a, nullptr); + store.release_assert_success (status); +} + +void nano::store::lmdb::pruned::del (store::write_transaction const & transaction_a, nano::block_hash const & hash_a) +{ + auto status = store.del (transaction_a, tables::pruned, hash_a); + store.release_assert_success (status); +} + +bool nano::store::lmdb::pruned::exists (store::transaction const & transaction_a, nano::block_hash const & hash_a) const +{ + return store.exists (transaction_a, tables::pruned, hash_a); +} + +nano::block_hash nano::store::lmdb::pruned::random (store::transaction const & transaction) +{ + nano::block_hash random_hash; + nano::random_pool::generate_block (random_hash.bytes.data (), random_hash.bytes.size ()); + auto existing = begin (transaction, random_hash); + if (existing == end ()) + { + existing = begin (transaction); + } + return existing != end () ? existing->first : 0; +} + +size_t nano::store::lmdb::pruned::count (store::transaction const & transaction_a) const +{ + return store.count (transaction_a, tables::pruned); +} + +void nano::store::lmdb::pruned::clear (store::write_transaction const & transaction_a) +{ + auto status = store.drop (transaction_a, tables::pruned); + store.release_assert_success (status); +} + +nano::store::iterator nano::store::lmdb::pruned::begin (store::transaction const & transaction, nano::block_hash const & hash) const +{ + return store.make_iterator (transaction, tables::pruned, hash); +} + +nano::store::iterator nano::store::lmdb::pruned::begin (store::transaction const & transaction) const +{ + return store.make_iterator (transaction, tables::pruned); +} + +nano::store::iterator nano::store::lmdb::pruned::end () const +{ + return store::iterator (nullptr); +} + +void nano::store::lmdb::pruned::for_each_par (std::function, store::iterator)> const & action_a) const +{ + parallel_traversal ( + [&action_a, this] (nano::uint256_t const & start, nano::uint256_t const & end, bool const is_last) { + auto transaction (this->store.tx_begin_read ()); + action_a (transaction, this->begin (transaction, start), !is_last ? this->begin (transaction, end) : this->end ()); + }); +} diff --git a/nano/store/lmdb/pruned.hpp b/nano/store/lmdb/pruned.hpp new file mode 100644 index 0000000000..d0c3c5d075 --- /dev/null +++ b/nano/store/lmdb/pruned.hpp @@ -0,0 +1,33 @@ +#pragma once + +#include + +#include + +namespace nano::store::lmdb +{ +class pruned : public nano::store::pruned +{ +private: + nano::store::lmdb::component & store; + +public: + explicit pruned (nano::store::lmdb::component & store_a); + void put (store::write_transaction const & transaction_a, nano::block_hash const & hash_a) override; + void del (store::write_transaction const & transaction_a, nano::block_hash const & hash_a) override; + bool exists (store::transaction const & transaction_a, nano::block_hash const & hash_a) const override; + nano::block_hash random (store::transaction const & transaction_a) override; + size_t count (store::transaction const & transaction_a) const override; + void clear (store::write_transaction const & transaction_a) override; + store::iterator begin (store::transaction const & transaction_a, nano::block_hash const & hash_a) const override; + store::iterator begin (store::transaction const & transaction_a) const override; + store::iterator end () const override; + void for_each_par (std::function, store::iterator)> const & action_a) const override; + + /** + * Pruned blocks hashes + * nano::block_hash -> none + */ + MDB_dbi pruned_handle{ 0 }; +}; +} // namespace nano::store::lmdb diff --git a/nano/node/lmdb/lmdb_txn.cpp b/nano/store/lmdb/transaction.cpp similarity index 80% rename from nano/node/lmdb/lmdb_txn.cpp rename to nano/store/lmdb/transaction.cpp index 5d577b384a..ad18863934 100644 --- a/nano/node/lmdb/lmdb_txn.cpp +++ b/nano/store/lmdb/transaction.cpp @@ -1,10 +1,10 @@ #include #include -#include +#include #include -#include -#include -#include +#include +#include +#include #include @@ -20,7 +20,7 @@ namespace class matches_txn final { public: - explicit matches_txn (nano::transaction_impl const * transaction_impl_a) : + explicit matches_txn (nano::store::transaction_impl const * transaction_impl_a) : transaction_impl (transaction_impl_a) { } @@ -31,11 +31,11 @@ class matches_txn final } private: - nano::transaction_impl const * transaction_impl; + nano::store::transaction_impl const * transaction_impl; }; } -nano::read_mdb_txn::read_mdb_txn (nano::mdb_env const & environment_a, nano::mdb_txn_callbacks txn_callbacks_a) : +nano::store::lmdb::read_transaction_impl::read_transaction_impl (nano::store::lmdb::env const & environment_a, nano::store::lmdb::txn_callbacks txn_callbacks_a) : txn_callbacks (txn_callbacks_a) { auto status (mdb_txn_begin (environment_a, nullptr, MDB_RDONLY, &handle)); @@ -43,7 +43,7 @@ nano::read_mdb_txn::read_mdb_txn (nano::mdb_env const & environment_a, nano::mdb txn_callbacks.txn_start (this); } -nano::read_mdb_txn::~read_mdb_txn () +nano::store::lmdb::read_transaction_impl::~read_transaction_impl () { // This uses commit rather than abort, as it is needed when opening databases with a read only transaction auto status (mdb_txn_commit (handle)); @@ -51,37 +51,37 @@ nano::read_mdb_txn::~read_mdb_txn () txn_callbacks.txn_end (this); } -void nano::read_mdb_txn::reset () +void nano::store::lmdb::read_transaction_impl::reset () { mdb_txn_reset (handle); txn_callbacks.txn_end (this); } -void nano::read_mdb_txn::renew () +void nano::store::lmdb::read_transaction_impl::renew () { auto status (mdb_txn_renew (handle)); release_assert (status == 0); txn_callbacks.txn_start (this); } -void * nano::read_mdb_txn::get_handle () const +void * nano::store::lmdb::read_transaction_impl::get_handle () const { return handle; } -nano::write_mdb_txn::write_mdb_txn (nano::mdb_env const & environment_a, nano::mdb_txn_callbacks txn_callbacks_a) : +nano::store::lmdb::write_transaction_impl::write_transaction_impl (nano::store::lmdb::env const & environment_a, nano::store::lmdb::txn_callbacks txn_callbacks_a) : env (environment_a), txn_callbacks (txn_callbacks_a) { renew (); } -nano::write_mdb_txn::~write_mdb_txn () +nano::store::lmdb::write_transaction_impl::~write_transaction_impl () { commit (); } -void nano::write_mdb_txn::commit () +void nano::store::lmdb::write_transaction_impl::commit () { if (active) { @@ -95,7 +95,7 @@ void nano::write_mdb_txn::commit () } } -void nano::write_mdb_txn::renew () +void nano::store::lmdb::write_transaction_impl::renew () { auto status (mdb_txn_begin (env, nullptr, 0, &handle)); release_assert (status == MDB_SUCCESS, mdb_strerror (status)); @@ -103,12 +103,12 @@ void nano::write_mdb_txn::renew () active = true; } -void * nano::write_mdb_txn::get_handle () const +void * nano::store::lmdb::write_transaction_impl::get_handle () const { return handle; } -bool nano::write_mdb_txn::contains (nano::tables table_a) const +bool nano::store::lmdb::write_transaction_impl::contains (nano::tables table_a) const { // LMDB locks on every write return true; @@ -196,7 +196,7 @@ void nano::mdb_txn_tracker::log_if_held_long_enough (nano::mdb_txn_stats const & } } -void nano::mdb_txn_tracker::add (nano::transaction_impl const * transaction_impl) +void nano::mdb_txn_tracker::add (store::transaction_impl const * transaction_impl) { nano::lock_guard guard (mutex); debug_assert (std::find_if (stats.cbegin (), stats.cend (), matches_txn (transaction_impl)) == stats.cend ()); @@ -204,7 +204,7 @@ void nano::mdb_txn_tracker::add (nano::transaction_impl const * transaction_impl } /** Can be called without error if transaction does not exist */ -void nano::mdb_txn_tracker::erase (nano::transaction_impl const * transaction_impl) +void nano::mdb_txn_tracker::erase (store::transaction_impl const * transaction_impl) { nano::unique_lock lk (mutex); auto it = std::find_if (stats.begin (), stats.end (), matches_txn (transaction_impl)); @@ -217,7 +217,7 @@ void nano::mdb_txn_tracker::erase (nano::transaction_impl const * transaction_im } } -nano::mdb_txn_stats::mdb_txn_stats (nano::transaction_impl const * transaction_impl) : +nano::mdb_txn_stats::mdb_txn_stats (store::transaction_impl const * transaction_impl) : transaction_impl (transaction_impl), thread_name (nano::thread_role::get_string ()), stacktrace (std::make_shared ()) @@ -227,5 +227,5 @@ nano::mdb_txn_stats::mdb_txn_stats (nano::transaction_impl const * transaction_i bool nano::mdb_txn_stats::is_write () const { - return (dynamic_cast (transaction_impl) != nullptr); + return (dynamic_cast (transaction_impl) != nullptr); } diff --git a/nano/node/lmdb/lmdb_txn.hpp b/nano/store/lmdb/transaction_impl.hpp similarity index 56% rename from nano/node/lmdb/lmdb_txn.hpp rename to nano/store/lmdb/transaction_impl.hpp index 884da8b4fb..cdeff1404e 100644 --- a/nano/node/lmdb/lmdb_txn.hpp +++ b/nano/store/lmdb/transaction_impl.hpp @@ -2,7 +2,8 @@ #include #include -#include +#include +#include #include #include @@ -11,51 +12,58 @@ namespace nano { -class transaction_impl; class logger_mt; -class mdb_env; +} +namespace nano::store::lmdb +{ +class env; +} -class mdb_txn_callbacks +namespace nano::store::lmdb +{ +class txn_callbacks { public: - std::function txn_start{ [] (nano::transaction_impl const *) {} }; - std::function txn_end{ [] (nano::transaction_impl const *) {} }; + std::function txn_start{ [] (store::transaction_impl const *) {} }; + std::function txn_end{ [] (store::transaction_impl const *) {} }; }; - -class read_mdb_txn final : public read_transaction_impl +class read_transaction_impl final : public store::read_transaction_impl { public: - read_mdb_txn (nano::mdb_env const &, mdb_txn_callbacks mdb_txn_callbacks); - ~read_mdb_txn (); + read_transaction_impl (nano::store::lmdb::env const &, txn_callbacks mdb_txn_callbacks); + ~read_transaction_impl (); void reset () override; void renew () override; void * get_handle () const override; MDB_txn * handle; - mdb_txn_callbacks txn_callbacks; + lmdb::txn_callbacks txn_callbacks; }; -class write_mdb_txn final : public write_transaction_impl +class write_transaction_impl final : public store::write_transaction_impl { public: - write_mdb_txn (nano::mdb_env const &, mdb_txn_callbacks mdb_txn_callbacks); - ~write_mdb_txn (); + write_transaction_impl (nano::store::lmdb::env const &, txn_callbacks mdb_txn_callbacks); + ~write_transaction_impl (); void commit () override; void renew () override; void * get_handle () const override; bool contains (nano::tables table_a) const override; MDB_txn * handle; - nano::mdb_env const & env; - mdb_txn_callbacks txn_callbacks; + nano::store::lmdb::env const & env; + lmdb::txn_callbacks txn_callbacks; bool active{ true }; }; +} // namespace nano +namespace nano +{ class mdb_txn_stats { public: - mdb_txn_stats (nano::transaction_impl const * transaction_impl_a); + mdb_txn_stats (store::transaction_impl const * transaction_impl_a); bool is_write () const; nano::timer timer; - nano::transaction_impl const * transaction_impl; + store::transaction_impl const * transaction_impl; std::string thread_name; // Smart pointer so that we don't need the full definition which causes min/max issues on Windows @@ -67,8 +75,8 @@ class mdb_txn_tracker public: mdb_txn_tracker (nano::logger_mt & logger_a, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a); void serialize_json (boost::property_tree::ptree & json, std::chrono::milliseconds min_read_time, std::chrono::milliseconds min_write_time); - void add (nano::transaction_impl const * transaction_impl); - void erase (nano::transaction_impl const * transaction_impl); + void add (store::transaction_impl const * transaction_impl); + void erase (store::transaction_impl const * transaction_impl); private: nano::mutex mutex; @@ -79,4 +87,4 @@ class mdb_txn_tracker void log_if_held_long_enough (nano::mdb_txn_stats const & mdb_txn_stats) const; }; -} +} // namespace nano diff --git a/nano/node/lmdb/version_store.cpp b/nano/store/lmdb/version.cpp similarity index 63% rename from nano/node/lmdb/version_store.cpp rename to nano/store/lmdb/version.cpp index deee22ef67..f667dc4b9c 100644 --- a/nano/node/lmdb/version_store.cpp +++ b/nano/store/lmdb/version.cpp @@ -1,10 +1,10 @@ -#include -#include +#include +#include -nano::lmdb::version_store::version_store (nano::lmdb::store & store_a) : +nano::store::lmdb::version::version (nano::store::lmdb::component & store_a) : store{ store_a } {}; -void nano::lmdb::version_store::put (nano::write_transaction const & transaction_a, int version) +void nano::store::lmdb::version::put (store::write_transaction const & transaction_a, int version) { nano::uint256_union version_key{ 1 }; nano::uint256_union version_value (version); @@ -12,10 +12,10 @@ void nano::lmdb::version_store::put (nano::write_transaction const & transaction store.release_assert_success (status); } -int nano::lmdb::version_store::get (nano::transaction const & transaction_a) const +int nano::store::lmdb::version::get (store::transaction const & transaction_a) const { nano::uint256_union version_key{ 1 }; - nano::mdb_val data; + nano::store::lmdb::db_val data; auto status = store.get (transaction_a, tables::meta, version_key, data); int result = store.version_minimum; if (store.success (status)) diff --git a/nano/store/lmdb/version.hpp b/nano/store/lmdb/version.hpp new file mode 100644 index 0000000000..fa9cb04298 --- /dev/null +++ b/nano/store/lmdb/version.hpp @@ -0,0 +1,25 @@ +#pragma once + +#include + +#include + +namespace nano::store::lmdb +{ +class version : public nano::store::version +{ +protected: + nano::store::lmdb::component & store; + +public: + explicit version (nano::store::lmdb::component & store_a); + void put (store::write_transaction const & transaction_a, int version_a) override; + int get (store::transaction const & transaction_a) const override; + + /** + * Meta information about block store, such as versions. + * nano::uint256_union (arbitrary key) -> blob + */ + MDB_dbi meta_handle{ 0 }; +}; +} // namespace nano::store::lmdb diff --git a/nano/node/lmdb/wallet_value.cpp b/nano/store/lmdb/wallet_value.cpp similarity index 68% rename from nano/node/lmdb/wallet_value.cpp rename to nano/store/lmdb/wallet_value.cpp index c9512df636..60aefabe25 100644 --- a/nano/node/lmdb/wallet_value.cpp +++ b/nano/store/lmdb/wallet_value.cpp @@ -1,6 +1,6 @@ -#include +#include -nano::wallet_value::wallet_value (nano::db_val const & val_a) +nano::wallet_value::wallet_value (nano::store::db_val const & val_a) { debug_assert (val_a.size () == sizeof (*this)); std::copy (reinterpret_cast (val_a.data ()), reinterpret_cast (val_a.data ()) + sizeof (key), key.chars.begin ()); @@ -13,8 +13,8 @@ nano::wallet_value::wallet_value (nano::raw_key const & key_a, uint64_t work_a) { } -nano::db_val nano::wallet_value::val () const +nano::store::db_val nano::wallet_value::val () const { static_assert (sizeof (*this) == sizeof (key) + sizeof (work), "Class not packed"); - return nano::db_val (sizeof (*this), const_cast (this)); + return nano::store::db_val (sizeof (*this), const_cast (this)); } diff --git a/nano/node/lmdb/wallet_value.hpp b/nano/store/lmdb/wallet_value.hpp similarity index 68% rename from nano/node/lmdb/wallet_value.hpp rename to nano/store/lmdb/wallet_value.hpp index 929ff13d0a..d272ae96d4 100644 --- a/nano/node/lmdb/wallet_value.hpp +++ b/nano/store/lmdb/wallet_value.hpp @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include @@ -11,9 +11,9 @@ class wallet_value { public: wallet_value () = default; - wallet_value (nano::db_val const &); + wallet_value (store::db_val const &); wallet_value (nano::raw_key const &, uint64_t); - nano::db_val val () const; + store::db_val val () const; nano::raw_key key; uint64_t work; }; diff --git a/nano/store/online_weight.cpp b/nano/store/online_weight.cpp new file mode 100644 index 0000000000..230a4f266d --- /dev/null +++ b/nano/store/online_weight.cpp @@ -0,0 +1 @@ +#include diff --git a/nano/store/online_weight.hpp b/nano/store/online_weight.hpp new file mode 100644 index 0000000000..aa17938e86 --- /dev/null +++ b/nano/store/online_weight.hpp @@ -0,0 +1,29 @@ +#pragma once + +#include +#include +#include + +#include + +namespace nano +{ +class block_hash; +} +namespace nano::store +{ +/** + * Manages online weight storage and iteration + */ +class online_weight +{ +public: + virtual void put (store::write_transaction const &, uint64_t, nano::amount const &) = 0; + virtual void del (store::write_transaction const &, uint64_t) = 0; + virtual store::iterator begin (store::transaction const &) const = 0; + virtual store::iterator rbegin (store::transaction const &) const = 0; + virtual store::iterator end () const = 0; + virtual size_t count (store::transaction const &) const = 0; + virtual void clear (store::write_transaction const &) = 0; +}; +} // namespace nano::store diff --git a/nano/store/peer.cpp b/nano/store/peer.cpp new file mode 100644 index 0000000000..e442b57549 --- /dev/null +++ b/nano/store/peer.cpp @@ -0,0 +1 @@ +#include diff --git a/nano/store/peer.hpp b/nano/store/peer.hpp new file mode 100644 index 0000000000..5f2aeb9766 --- /dev/null +++ b/nano/store/peer.hpp @@ -0,0 +1,29 @@ +#pragma once + +#include +#include +#include + +#include + +namespace nano +{ +class block_hash; +} +namespace nano::store +{ +/** + * Manages peer storage and iteration + */ +class peer +{ +public: + virtual void put (store::write_transaction const & transaction_a, nano::endpoint_key const & endpoint_a) = 0; + virtual void del (store::write_transaction const & transaction_a, nano::endpoint_key const & endpoint_a) = 0; + virtual bool exists (store::transaction const & transaction_a, nano::endpoint_key const & endpoint_a) const = 0; + virtual size_t count (store::transaction const & transaction_a) const = 0; + virtual void clear (store::write_transaction const & transaction_a) = 0; + virtual store::iterator begin (store::transaction const & transaction_a) const = 0; + virtual store::iterator end () const = 0; +}; +} // namespace nano::store diff --git a/nano/store/pending.cpp b/nano/store/pending.cpp new file mode 100644 index 0000000000..dd80f3579b --- /dev/null +++ b/nano/store/pending.cpp @@ -0,0 +1 @@ +#include diff --git a/nano/store/pending.hpp b/nano/store/pending.hpp new file mode 100644 index 0000000000..6c5b379323 --- /dev/null +++ b/nano/store/pending.hpp @@ -0,0 +1,31 @@ +#pragma once + +#include +#include +#include + +#include + +namespace nano +{ +class block_hash; +} +namespace nano::store +{ +/** + * Manages pending storage and iteration + */ +class pending +{ +public: + virtual void put (store::write_transaction const &, nano::pending_key const &, nano::pending_info const &) = 0; + virtual void del (store::write_transaction const &, nano::pending_key const &) = 0; + virtual bool get (store::transaction const &, nano::pending_key const &, nano::pending_info &) = 0; + virtual bool exists (store::transaction const &, nano::pending_key const &) = 0; + virtual bool any (store::transaction const &, nano::account const &) = 0; + virtual store::iterator begin (store::transaction const &, nano::pending_key const &) const = 0; + virtual store::iterator begin (store::transaction const &) const = 0; + virtual store::iterator end () const = 0; + virtual void for_each_par (std::function, store::iterator)> const & action_a) const = 0; +}; +} // namespace nano::store diff --git a/nano/store/pruned.cpp b/nano/store/pruned.cpp new file mode 100644 index 0000000000..e4342c7b4d --- /dev/null +++ b/nano/store/pruned.cpp @@ -0,0 +1 @@ +#include diff --git a/nano/store/pruned.hpp b/nano/store/pruned.hpp new file mode 100644 index 0000000000..cc40c812b8 --- /dev/null +++ b/nano/store/pruned.hpp @@ -0,0 +1,32 @@ +#pragma once + +#include +#include +#include + +#include + +namespace nano +{ +class block_hash; +} +namespace nano::store +{ +/** + * Manages pruned storage and iteration + */ +class pruned +{ +public: + virtual void put (store::write_transaction const & transaction_a, nano::block_hash const & hash_a) = 0; + virtual void del (store::write_transaction const & transaction_a, nano::block_hash const & hash_a) = 0; + virtual bool exists (store::transaction const & transaction_a, nano::block_hash const & hash_a) const = 0; + virtual nano::block_hash random (store::transaction const & transaction_a) = 0; + virtual size_t count (store::transaction const & transaction_a) const = 0; + virtual void clear (store::write_transaction const &) = 0; + virtual store::iterator begin (store::transaction const & transaction_a, nano::block_hash const & hash_a) const = 0; + virtual store::iterator begin (store::transaction const & transaction_a) const = 0; + virtual store::iterator end () const = 0; + virtual void for_each_par (std::function, store::iterator)> const & action_a) const = 0; +}; +} // namespace nano::store diff --git a/nano/store/rocksdb/account.cpp b/nano/store/rocksdb/account.cpp new file mode 100644 index 0000000000..8ee01b98fe --- /dev/null +++ b/nano/store/rocksdb/account.cpp @@ -0,0 +1,72 @@ +#include +#include +#include + +nano::store::rocksdb::account::account (nano::store::rocksdb::component & store_a) : + store (store_a){}; + +void nano::store::rocksdb::account::put (store::write_transaction const & transaction, nano::account const & account, nano::account_info const & info) +{ + auto status = store.put (transaction, tables::accounts, account, info); + store.release_assert_success (status); +} + +bool nano::store::rocksdb::account::get (store::transaction const & transaction, nano::account const & account, nano::account_info & info) +{ + nano::store::rocksdb::db_val value; + auto status1 (store.get (transaction, tables::accounts, account, value)); + release_assert (store.success (status1) || store.not_found (status1)); + bool result (true); + if (store.success (status1)) + { + nano::bufferstream stream (reinterpret_cast (value.data ()), value.size ()); + result = info.deserialize (stream); + } + return result; +} + +void nano::store::rocksdb::account::del (store::write_transaction const & transaction_a, nano::account const & account_a) +{ + auto status = store.del (transaction_a, tables::accounts, account_a); + store.release_assert_success (status); +} + +bool nano::store::rocksdb::account::exists (store::transaction const & transaction_a, nano::account const & account_a) +{ + auto iterator (begin (transaction_a, account_a)); + return iterator != end () && nano::account (iterator->first) == account_a; +} + +size_t nano::store::rocksdb::account::count (store::transaction const & transaction_a) +{ + return store.count (transaction_a, tables::accounts); +} + +nano::store::iterator nano::store::rocksdb::account::begin (store::transaction const & transaction, nano::account const & account) const +{ + return store.make_iterator (transaction, tables::accounts, account); +} + +nano::store::iterator nano::store::rocksdb::account::begin (store::transaction const & transaction) const +{ + return store.make_iterator (transaction, tables::accounts); +} + +nano::store::iterator nano::store::rocksdb::account::rbegin (store::transaction const & transaction_a) const +{ + return store.make_iterator (transaction_a, tables::accounts, false); +} + +nano::store::iterator nano::store::rocksdb::account::end () const +{ + return store::iterator (nullptr); +} + +void nano::store::rocksdb::account::for_each_par (std::function, store::iterator)> const & action_a) const +{ + parallel_traversal ( + [&action_a, this] (nano::uint256_t const & start, nano::uint256_t const & end, bool const is_last) { + auto transaction (this->store.tx_begin_read ()); + action_a (transaction, this->begin (transaction, start), !is_last ? this->begin (transaction, end) : this->end ()); + }); +} diff --git a/nano/store/rocksdb/account.hpp b/nano/store/rocksdb/account.hpp new file mode 100644 index 0000000000..df5752b4fc --- /dev/null +++ b/nano/store/rocksdb/account.hpp @@ -0,0 +1,29 @@ +#pragma once + +#include + +namespace nano::store::rocksdb +{ +class component; +} +namespace nano::store::rocksdb +{ +class account : public nano::store::account +{ +private: + nano::store::rocksdb::component & store; + +public: + explicit account (nano::store::rocksdb::component & store_a); + void put (store::write_transaction const & transaction, nano::account const & account, nano::account_info const & info) override; + bool get (store::transaction const & transaction_a, nano::account const & account_a, nano::account_info & info_a) override; + void del (store::write_transaction const & transaction_a, nano::account const & account_a) override; + bool exists (store::transaction const & transaction_a, nano::account const & account_a) override; + size_t count (store::transaction const & transaction_a) override; + store::iterator begin (store::transaction const & transaction_a, nano::account const & account_a) const override; + store::iterator begin (store::transaction const & transaction_a) const override; + store::iterator rbegin (store::transaction const & transaction_a) const override; + store::iterator end () const override; + void for_each_par (std::function, store::iterator)> const & action_a) const override; +}; +} // namespace nano::store::rocksdb diff --git a/nano/store/rocksdb/block.cpp b/nano/store/rocksdb/block.cpp new file mode 100644 index 0000000000..fbf1ecf1f0 --- /dev/null +++ b/nano/store/rocksdb/block.cpp @@ -0,0 +1,212 @@ +#include +#include +#include + +namespace nano +{ +/** + * Fill in our predecessors + */ +class block_predecessor_rocksdb_set : public nano::block_visitor +{ +public: + block_predecessor_rocksdb_set (store::write_transaction const & transaction_a, nano::store::rocksdb::block & block_store_a); + virtual ~block_predecessor_rocksdb_set () = default; + void fill_value (nano::block const & block_a); + void send_block (nano::send_block const & block_a) override; + void receive_block (nano::receive_block const & block_a) override; + void open_block (nano::open_block const & block_a) override; + void change_block (nano::change_block const & block_a) override; + void state_block (nano::state_block const & block_a) override; + store::write_transaction const & transaction; + nano::store::rocksdb::block & block_store; +}; +} + +nano::store::rocksdb::block::block (nano::store::rocksdb::component & store_a) : + store{ store_a } {}; + +void nano::store::rocksdb::block::put (store::write_transaction const & transaction, nano::block_hash const & hash, nano::block const & block) +{ + debug_assert (block.sideband ().successor.is_zero () || exists (transaction, block.sideband ().successor)); + std::vector vector; + { + nano::vectorstream stream (vector); + nano::serialize_block (stream, block); + block.sideband ().serialize (stream, block.type ()); + } + raw_put (transaction, vector, hash); + block_predecessor_rocksdb_set predecessor (transaction, *this); + block.visit (predecessor); + debug_assert (block.previous ().is_zero () || successor (transaction, block.previous ()) == hash); +} + +void nano::store::rocksdb::block::raw_put (store::write_transaction const & transaction_a, std::vector const & data, nano::block_hash const & hash_a) +{ + nano::store::rocksdb::db_val value{ data.size (), (void *)data.data () }; + auto status = store.put (transaction_a, tables::blocks, hash_a, value); + store.release_assert_success (status); +} + +nano::block_hash nano::store::rocksdb::block::successor (store::transaction const & transaction_a, nano::block_hash const & hash_a) const +{ + nano::store::rocksdb::db_val value; + block_raw_get (transaction_a, hash_a, value); + nano::block_hash result; + if (value.size () != 0) + { + debug_assert (value.size () >= result.bytes.size ()); + auto type = block_type_from_raw (value.data ()); + nano::bufferstream stream (reinterpret_cast (value.data ()) + block_successor_offset (transaction_a, value.size (), type), result.bytes.size ()); + auto error (nano::try_read (stream, result.bytes)); + (void)error; + debug_assert (!error); + } + else + { + result.clear (); + } + return result; +} + +void nano::store::rocksdb::block::successor_clear (store::write_transaction const & transaction, nano::block_hash const & hash) +{ + nano::store::rocksdb::db_val value; + block_raw_get (transaction, hash, value); + debug_assert (value.size () != 0); + auto type = block_type_from_raw (value.data ()); + std::vector data (static_cast (value.data ()), static_cast (value.data ()) + value.size ()); + std::fill_n (data.begin () + block_successor_offset (transaction, value.size (), type), sizeof (nano::block_hash), uint8_t{ 0 }); + raw_put (transaction, data, hash); +} + +std::shared_ptr nano::store::rocksdb::block::get (store::transaction const & transaction, nano::block_hash const & hash) const +{ + nano::store::rocksdb::db_val value; + block_raw_get (transaction, hash, value); + std::shared_ptr result; + if (value.size () != 0) + { + nano::bufferstream stream (reinterpret_cast (value.data ()), value.size ()); + nano::block_type type; + auto error (try_read (stream, type)); + release_assert (!error); + result = nano::deserialize_block (stream, type); + release_assert (result != nullptr); + nano::block_sideband sideband; + error = (sideband.deserialize (stream, type)); + release_assert (!error); + result->sideband_set (sideband); + } + return result; +} +std::shared_ptr nano::store::rocksdb::block::random (store::transaction const & transaction) +{ + nano::block_hash hash; + nano::random_pool::generate_block (hash.bytes.data (), hash.bytes.size ()); + auto existing = begin (transaction, hash); + if (existing == end ()) + { + existing = begin (transaction); + } + debug_assert (existing != end ()); + return existing->second.block; +} + +void nano::store::rocksdb::block::del (store::write_transaction const & transaction_a, nano::block_hash const & hash_a) +{ + auto status = store.del (transaction_a, tables::blocks, hash_a); + store.release_assert_success (status); +} + +bool nano::store::rocksdb::block::exists (store::transaction const & transaction, nano::block_hash const & hash) +{ + nano::store::rocksdb::db_val junk; + block_raw_get (transaction, hash, junk); + return junk.size () != 0; +} + +uint64_t nano::store::rocksdb::block::count (store::transaction const & transaction_a) +{ + return store.count (transaction_a, tables::blocks); +} +nano::store::iterator nano::store::rocksdb::block::begin (store::transaction const & transaction) const +{ + return store.make_iterator (transaction, tables::blocks); +} + +nano::store::iterator nano::store::rocksdb::block::begin (store::transaction const & transaction, nano::block_hash const & hash) const +{ + return store.make_iterator (transaction, tables::blocks, hash); +} + +nano::store::iterator nano::store::rocksdb::block::end () const +{ + return store::iterator (nullptr); +} + +void nano::store::rocksdb::block::for_each_par (std::function, store::iterator)> const & action_a) const +{ + parallel_traversal ( + [&action_a, this] (nano::uint256_t const & start, nano::uint256_t const & end, bool const is_last) { + auto transaction (this->store.tx_begin_read ()); + action_a (transaction, this->begin (transaction, start), !is_last ? this->begin (transaction, end) : this->end ()); + }); +} + +void nano::store::rocksdb::block::block_raw_get (store::transaction const & transaction, nano::block_hash const & hash, nano::store::rocksdb::db_val & value) const +{ + auto status = store.get (transaction, tables::blocks, hash, value); + release_assert (store.success (status) || store.not_found (status)); +} + +size_t nano::store::rocksdb::block::block_successor_offset (store::transaction const & transaction_a, size_t entry_size_a, nano::block_type type_a) const +{ + return entry_size_a - nano::block_sideband::size (type_a); +} + +nano::block_type nano::store::rocksdb::block::block_type_from_raw (void * data_a) +{ + // The block type is the first byte + return static_cast ((reinterpret_cast (data_a))[0]); +} + +nano::block_predecessor_rocksdb_set::block_predecessor_rocksdb_set (store::write_transaction const & transaction_a, nano::store::rocksdb::block & block_store_a) : + transaction{ transaction_a }, + block_store{ block_store_a } +{ +} +void nano::block_predecessor_rocksdb_set::fill_value (nano::block const & block_a) +{ + auto hash = block_a.hash (); + nano::store::rocksdb::db_val value; + block_store.block_raw_get (transaction, block_a.previous (), value); + debug_assert (value.size () != 0); + auto type = block_store.block_type_from_raw (value.data ()); + std::vector data (static_cast (value.data ()), static_cast (value.data ()) + value.size ()); + std::copy (hash.bytes.begin (), hash.bytes.end (), data.begin () + block_store.block_successor_offset (transaction, value.size (), type)); + block_store.raw_put (transaction, data, block_a.previous ()); +} +void nano::block_predecessor_rocksdb_set::send_block (nano::send_block const & block_a) +{ + fill_value (block_a); +} +void nano::block_predecessor_rocksdb_set::receive_block (nano::receive_block const & block_a) +{ + fill_value (block_a); +} +void nano::block_predecessor_rocksdb_set::open_block (nano::open_block const & block_a) +{ + // Open blocks don't have a predecessor +} +void nano::block_predecessor_rocksdb_set::change_block (nano::change_block const & block_a) +{ + fill_value (block_a); +} +void nano::block_predecessor_rocksdb_set::state_block (nano::state_block const & block_a) +{ + if (!block_a.previous ().is_zero ()) + { + fill_value (block_a); + } +} diff --git a/nano/store/rocksdb/block.hpp b/nano/store/rocksdb/block.hpp new file mode 100644 index 0000000000..6ecf26fd62 --- /dev/null +++ b/nano/store/rocksdb/block.hpp @@ -0,0 +1,42 @@ +#pragma once + +#include +#include + +namespace nano +{ +class block_predecessor_rocksdb_set; +} +namespace nano::store::rocksdb +{ +class component; +} +namespace nano::store::rocksdb +{ +class block : public nano::store::block +{ + friend class nano::block_predecessor_rocksdb_set; + nano::store::rocksdb::component & store; + +public: + explicit block (nano::store::rocksdb::component & store_a); + void put (store::write_transaction const & transaction_a, nano::block_hash const & hash_a, nano::block const & block_a) override; + void raw_put (store::write_transaction const & transaction_a, std::vector const & data, nano::block_hash const & hash_a) override; + nano::block_hash successor (store::transaction const & transaction_a, nano::block_hash const & hash_a) const override; + void successor_clear (store::write_transaction const & transaction_a, nano::block_hash const & hash_a) override; + std::shared_ptr get (store::transaction const & transaction_a, nano::block_hash const & hash_a) const override; + std::shared_ptr random (store::transaction const & transaction_a) override; + void del (store::write_transaction const & transaction_a, nano::block_hash const & hash_a) override; + bool exists (store::transaction const & transaction_a, nano::block_hash const & hash_a) override; + uint64_t count (store::transaction const & transaction_a) override; + store::iterator begin (store::transaction const & transaction_a) const override; + store::iterator begin (store::transaction const & transaction_a, nano::block_hash const & hash_a) const override; + store::iterator end () const override; + void for_each_par (std::function, store::iterator)> const & action_a) const override; + +protected: + void block_raw_get (store::transaction const & transaction_a, nano::block_hash const & hash_a, nano::store::rocksdb::db_val & value) const; + size_t block_successor_offset (store::transaction const & transaction_a, size_t entry_size_a, nano::block_type type_a) const; + static nano::block_type block_type_from_raw (void * data_a); +}; +} // namespace nano::store::rocksdb diff --git a/nano/store/rocksdb/confirmation_height.cpp b/nano/store/rocksdb/confirmation_height.cpp new file mode 100644 index 0000000000..380cd80e32 --- /dev/null +++ b/nano/store/rocksdb/confirmation_height.cpp @@ -0,0 +1,84 @@ +#include +#include +#include + +nano::store::rocksdb::confirmation_height::confirmation_height (nano::store::rocksdb::component & store) : + store{ store } +{ +} + +void nano::store::rocksdb::confirmation_height::put (store::write_transaction const & transaction, nano::account const & account, nano::confirmation_height_info const & confirmation_height_info) +{ + auto status = store.put (transaction, tables::confirmation_height, account, confirmation_height_info); + store.release_assert_success (status); +} + +bool nano::store::rocksdb::confirmation_height::get (store::transaction const & transaction, nano::account const & account, nano::confirmation_height_info & confirmation_height_info) +{ + nano::store::rocksdb::db_val value; + auto status = store.get (transaction, tables::confirmation_height, account, value); + release_assert (store.success (status) || store.not_found (status)); + bool result (true); + if (store.success (status)) + { + nano::bufferstream stream (reinterpret_cast (value.data ()), value.size ()); + result = confirmation_height_info.deserialize (stream); + } + if (result) + { + confirmation_height_info.height = 0; + confirmation_height_info.frontier = 0; + } + + return result; +} + +bool nano::store::rocksdb::confirmation_height::exists (store::transaction const & transaction, nano::account const & account) const +{ + return store.exists (transaction, tables::confirmation_height, account); +} + +void nano::store::rocksdb::confirmation_height::del (store::write_transaction const & transaction, nano::account const & account) +{ + auto status = store.del (transaction, tables::confirmation_height, account); + store.release_assert_success (status); +} + +uint64_t nano::store::rocksdb::confirmation_height::count (store::transaction const & transaction) +{ + return store.count (transaction, tables::confirmation_height); +} + +void nano::store::rocksdb::confirmation_height::clear (store::write_transaction const & transaction, nano::account const & account) +{ + del (transaction, account); +} + +void nano::store::rocksdb::confirmation_height::clear (store::write_transaction const & transaction) +{ + store.drop (transaction, nano::tables::confirmation_height); +} + +nano::store::iterator nano::store::rocksdb::confirmation_height::begin (store::transaction const & transaction, nano::account const & account) const +{ + return store.make_iterator (transaction, tables::confirmation_height, account); +} + +nano::store::iterator nano::store::rocksdb::confirmation_height::begin (store::transaction const & transaction) const +{ + return store.make_iterator (transaction, tables::confirmation_height); +} + +nano::store::iterator nano::store::rocksdb::confirmation_height::end () const +{ + return store::iterator (nullptr); +} + +void nano::store::rocksdb::confirmation_height::for_each_par (std::function, store::iterator)> const & action_a) const +{ + parallel_traversal ( + [&action_a, this] (nano::uint256_t const & start, nano::uint256_t const & end, bool const is_last) { + auto transaction (this->store.tx_begin_read ()); + action_a (transaction, this->begin (transaction, start), !is_last ? this->begin (transaction, end) : this->end ()); + }); +} diff --git a/nano/store/rocksdb/confirmation_height.hpp b/nano/store/rocksdb/confirmation_height.hpp new file mode 100644 index 0000000000..95d7300468 --- /dev/null +++ b/nano/store/rocksdb/confirmation_height.hpp @@ -0,0 +1,29 @@ +#pragma once + +#include + +namespace nano::store::rocksdb +{ +class component; +} +namespace nano::store::rocksdb +{ +class confirmation_height : public nano::store::confirmation_height +{ + nano::store::rocksdb::component & store; + +public: + explicit confirmation_height (nano::store::rocksdb::component & store_a); + void put (store::write_transaction const & transaction_a, nano::account const & account_a, nano::confirmation_height_info const & confirmation_height_info_a) override; + bool get (store::transaction const & transaction_a, nano::account const & account_a, nano::confirmation_height_info & confirmation_height_info_a) override; + bool exists (store::transaction const & transaction_a, nano::account const & account_a) const override; + void del (store::write_transaction const & transaction_a, nano::account const & account_a) override; + uint64_t count (store::transaction const & transaction_a) override; + void clear (store::write_transaction const & transaction_a, nano::account const & account_a) override; + void clear (store::write_transaction const & transaction_a) override; + store::iterator begin (store::transaction const & transaction_a, nano::account const & account_a) const override; + store::iterator begin (store::transaction const & transaction_a) const override; + store::iterator end () const override; + void for_each_par (std::function, store::iterator)> const & action_a) const override; +}; +} // namespace nano::store::rocksdb diff --git a/nano/store/rocksdb/db_val.cpp b/nano/store/rocksdb/db_val.cpp new file mode 100644 index 0000000000..e31654bbad --- /dev/null +++ b/nano/store/rocksdb/db_val.cpp @@ -0,0 +1,25 @@ +#include + +template <> +void * nano::store::rocksdb::db_val::data () const +{ + return (void *)value.data (); +} + +template <> +std::size_t nano::store::rocksdb::db_val::size () const +{ + return value.size (); +} + +template <> +nano::store::rocksdb::db_val::db_val (std::size_t size_a, void * data_a) : + value (static_cast (data_a), size_a) +{ +} + +template <> +void nano::store::rocksdb::db_val::convert_buffer_to_value () +{ + value = ::rocksdb::Slice (reinterpret_cast (buffer->data ()), buffer->size ()); +} diff --git a/nano/store/rocksdb/db_val.hpp b/nano/store/rocksdb/db_val.hpp new file mode 100644 index 0000000000..9c1085ee57 --- /dev/null +++ b/nano/store/rocksdb/db_val.hpp @@ -0,0 +1,10 @@ +#pragma once + +#include + +#include + +namespace nano::store::rocksdb +{ +using db_val = store::db_val<::rocksdb::Slice>; +} diff --git a/nano/node/lmdb/final_vote_store.cpp b/nano/store/rocksdb/final_vote.cpp similarity index 53% rename from nano/node/lmdb/final_vote_store.cpp rename to nano/store/rocksdb/final_vote.cpp index 6b7dfccf76..121da33360 100644 --- a/nano/node/lmdb/final_vote_store.cpp +++ b/nano/store/rocksdb/final_vote.cpp @@ -1,13 +1,13 @@ -#include -#include #include +#include +#include -nano::lmdb::final_vote_store::final_vote_store (nano::lmdb::store & store) : +nano::store::rocksdb::final_vote::final_vote (nano::store::rocksdb::component & store) : store{ store } {}; -bool nano::lmdb::final_vote_store::put (nano::write_transaction const & transaction, nano::qualified_root const & root, nano::block_hash const & hash) +bool nano::store::rocksdb::final_vote::put (store::write_transaction const & transaction, nano::qualified_root const & root, nano::block_hash const & hash) { - nano::mdb_val value; + nano::store::rocksdb::db_val value; auto status = store.get (transaction, tables::final_votes, root, value); release_assert (store.success (status) || store.not_found (status)); bool result (true); @@ -23,7 +23,7 @@ bool nano::lmdb::final_vote_store::put (nano::write_transaction const & transact return result; } -std::vector nano::lmdb::final_vote_store::get (nano::transaction const & transaction, nano::root const & root_a) +std::vector nano::store::rocksdb::final_vote::get (store::transaction const & transaction, nano::root const & root_a) { std::vector result; nano::qualified_root key_start{ root_a.raw, 0 }; @@ -34,7 +34,7 @@ std::vector nano::lmdb::final_vote_store::get (nano::transacti return result; } -void nano::lmdb::final_vote_store::del (nano::write_transaction const & transaction, nano::root const & root) +void nano::store::rocksdb::final_vote::del (store::write_transaction const & transaction, nano::root const & root) { std::vector final_vote_qualified_roots; for (auto i = begin (transaction, nano::qualified_root{ root.raw, 0 }), n = end (); i != n && nano::qualified_root{ i->first }.root () == root; ++i) @@ -49,37 +49,37 @@ void nano::lmdb::final_vote_store::del (nano::write_transaction const & transact } } -size_t nano::lmdb::final_vote_store::count (nano::transaction const & transaction_a) const +size_t nano::store::rocksdb::final_vote::count (store::transaction const & transaction_a) const { return store.count (transaction_a, tables::final_votes); } -void nano::lmdb::final_vote_store::clear (nano::write_transaction const & transaction_a, nano::root const & root_a) +void nano::store::rocksdb::final_vote::clear (store::write_transaction const & transaction_a, nano::root const & root_a) { del (transaction_a, root_a); } -void nano::lmdb::final_vote_store::clear (nano::write_transaction const & transaction_a) +void nano::store::rocksdb::final_vote::clear (store::write_transaction const & transaction_a) { store.drop (transaction_a, nano::tables::final_votes); } -nano::store_iterator nano::lmdb::final_vote_store::begin (nano::transaction const & transaction, nano::qualified_root const & root) const +nano::store::iterator nano::store::rocksdb::final_vote::begin (store::transaction const & transaction, nano::qualified_root const & root) const { return store.make_iterator (transaction, tables::final_votes, root); } -nano::store_iterator nano::lmdb::final_vote_store::begin (nano::transaction const & transaction) const +nano::store::iterator nano::store::rocksdb::final_vote::begin (store::transaction const & transaction) const { return store.make_iterator (transaction, tables::final_votes); } -nano::store_iterator nano::lmdb::final_vote_store::end () const +nano::store::iterator nano::store::rocksdb::final_vote::end () const { - return nano::store_iterator (nullptr); + return store::iterator (nullptr); } -void nano::lmdb::final_vote_store::for_each_par (std::function, nano::store_iterator)> const & action_a) const +void nano::store::rocksdb::final_vote::for_each_par (std::function, store::iterator)> const & action_a) const { parallel_traversal ( [&action_a, this] (nano::uint512_t const & start, nano::uint512_t const & end, bool const is_last) { diff --git a/nano/store/rocksdb/final_vote.hpp b/nano/store/rocksdb/final_vote.hpp new file mode 100644 index 0000000000..d730ec08b8 --- /dev/null +++ b/nano/store/rocksdb/final_vote.hpp @@ -0,0 +1,29 @@ +#pragma once + +#include + +namespace nano::store::rocksdb +{ +class component; +} +namespace nano::store::rocksdb +{ +class final_vote : public nano::store::final_vote +{ +private: + nano::store::rocksdb::component & store; + +public: + explicit final_vote (nano::store::rocksdb::component & store); + bool put (store::write_transaction const & transaction_a, nano::qualified_root const & root_a, nano::block_hash const & hash_a) override; + std::vector get (store::transaction const & transaction_a, nano::root const & root_a) override; + void del (store::write_transaction const & transaction_a, nano::root const & root_a) override; + size_t count (store::transaction const & transaction_a) const override; + void clear (store::write_transaction const & transaction_a, nano::root const & root_a) override; + void clear (store::write_transaction const & transaction_a) override; + store::iterator begin (store::transaction const & transaction_a, nano::qualified_root const & root_a) const override; + store::iterator begin (store::transaction const & transaction_a) const override; + store::iterator end () const override; + void for_each_par (std::function, store::iterator)> const & action_a) const override; +}; +} // namespace nano::store::rocksdb diff --git a/nano/store/rocksdb/frontier.cpp b/nano/store/rocksdb/frontier.cpp new file mode 100644 index 0000000000..6724b93363 --- /dev/null +++ b/nano/store/rocksdb/frontier.cpp @@ -0,0 +1,57 @@ +#include +#include +#include + +nano::store::rocksdb::frontier::frontier (nano::store::rocksdb::component & store) : + store{ store } +{ +} + +void nano::store::rocksdb::frontier::put (store::write_transaction const & transaction, nano::block_hash const & block, nano::account const & account) +{ + auto status = store.put (transaction, tables::frontiers, block, account); + store.release_assert_success (status); +} + +nano::account nano::store::rocksdb::frontier::get (store::transaction const & transaction, nano::block_hash const & hash) const +{ + db_val value; + auto status = store.get (transaction, tables::frontiers, hash, value); + release_assert (store.success (status) || store.not_found (status)); + nano::account result{}; + if (store.success (status)) + { + result = static_cast (value); + } + return result; +} + +void nano::store::rocksdb::frontier::del (store::write_transaction const & transaction, nano::block_hash const & hash) +{ + auto status = store.del (transaction, tables::frontiers, hash); + store.release_assert_success (status); +} + +nano::store::iterator nano::store::rocksdb::frontier::begin (store::transaction const & transaction) const +{ + return store.make_iterator (transaction, tables::frontiers); +} + +nano::store::iterator nano::store::rocksdb::frontier::begin (store::transaction const & transaction, nano::block_hash const & hash) const +{ + return store.make_iterator (transaction, tables::frontiers, hash); +} + +nano::store::iterator nano::store::rocksdb::frontier::end () const +{ + return nano::store::iterator (nullptr); +} + +void nano::store::rocksdb::frontier::for_each_par (std::function, nano::store::iterator)> const & action_a) const +{ + parallel_traversal ( + [&action_a, this] (nano::uint256_t const & start, nano::uint256_t const & end, bool const is_last) { + auto transaction (this->store.tx_begin_read ()); + action_a (transaction, this->begin (transaction, start), !is_last ? this->begin (transaction, end) : this->end ()); + }); +} diff --git a/nano/store/rocksdb/frontier.hpp b/nano/store/rocksdb/frontier.hpp new file mode 100644 index 0000000000..edc0e15b49 --- /dev/null +++ b/nano/store/rocksdb/frontier.hpp @@ -0,0 +1,27 @@ +#pragma once + +#include +#include + +namespace nano::store::rocksdb +{ +class component; +} +namespace nano::store::rocksdb +{ +class frontier : public nano::store::frontier +{ +public: + frontier (nano::store::rocksdb::component & store); + void put (store::write_transaction const &, nano::block_hash const &, nano::account const &) override; + nano::account get (store::transaction const &, nano::block_hash const &) const override; + void del (store::write_transaction const &, nano::block_hash const &) override; + store::iterator begin (store::transaction const &) const override; + store::iterator begin (store::transaction const &, nano::block_hash const &) const override; + store::iterator end () const override; + void for_each_par (std::function, store::iterator)> const & action_a) const override; + +private: + nano::store::rocksdb::component & store; +}; +} // namespace nano::store::rocksdb diff --git a/nano/node/rocksdb/rocksdb_iterator.hpp b/nano/store/rocksdb/iterator.hpp similarity index 63% rename from nano/node/rocksdb/rocksdb_iterator.hpp rename to nano/store/rocksdb/iterator.hpp index c4d4013700..0351367828 100644 --- a/nano/node/rocksdb/rocksdb_iterator.hpp +++ b/nano/store/rocksdb/iterator.hpp @@ -1,6 +1,9 @@ #pragma once -#include +#include +#include +#include +#include #include #include @@ -10,29 +13,27 @@ namespace { -inline bool is_read (nano::transaction const & transaction_a) +inline bool is_read (nano::store::transaction const & transaction_a) { - return (dynamic_cast (&transaction_a) != nullptr); + return (dynamic_cast (&transaction_a) != nullptr); } -inline rocksdb::ReadOptions & snapshot_options (nano::transaction const & transaction_a) +inline rocksdb::ReadOptions & snapshot_options (nano::store::transaction const & transaction_a) { debug_assert (is_read (transaction_a)); return *static_cast (transaction_a.get_handle ()); } } -namespace nano +namespace nano::store::rocksdb { -using rocksdb_val = db_val<::rocksdb::Slice>; - template -class rocksdb_iterator : public store_iterator_impl +class iterator : public iterator_impl { public: - rocksdb_iterator () = default; + iterator () = default; - rocksdb_iterator (::rocksdb::DB * db, nano::transaction const & transaction_a, ::rocksdb::ColumnFamilyHandle * handle_a, rocksdb_val const * val_a, bool const direction_asc) + iterator (::rocksdb::DB * db, store::transaction const & transaction_a, ::rocksdb::ColumnFamilyHandle * handle_a, db_val const * val_a, bool const direction_asc) { // Don't fill the block cache for any blocks read as a result of an iterator if (is_read (transaction_a)) @@ -72,21 +73,21 @@ class rocksdb_iterator : public store_iterator_impl } } - rocksdb_iterator (::rocksdb::DB * db, nano::transaction const & transaction_a, ::rocksdb::ColumnFamilyHandle * handle_a) : - rocksdb_iterator (db, transaction_a, handle_a, nullptr) + iterator (::rocksdb::DB * db, store::transaction const & transaction_a, ::rocksdb::ColumnFamilyHandle * handle_a) : + iterator (db, transaction_a, handle_a, nullptr) { } - rocksdb_iterator (nano::rocksdb_iterator && other_a) + iterator (nano::store::rocksdb::iterator && other_a) { cursor = other_a.cursor; other_a.cursor = nullptr; current = other_a.current; } - rocksdb_iterator (nano::rocksdb_iterator const &) = delete; + iterator (nano::store::rocksdb::iterator const &) = delete; - nano::store_iterator_impl & operator++ () override + store::iterator_impl & operator++ () override { cursor->Next (); if (cursor->Valid ()) @@ -107,7 +108,7 @@ class rocksdb_iterator : public store_iterator_impl return *this; } - nano::store_iterator_impl & operator-- () override + store::iterator_impl & operator-- () override { cursor->Prev (); if (cursor->Valid ()) @@ -128,14 +129,14 @@ class rocksdb_iterator : public store_iterator_impl return *this; } - std::pair * operator-> () + std::pair * operator-> () { return ¤t; } - bool operator== (nano::store_iterator_impl const & base_a) const override + bool operator== (store::iterator_impl const & base_a) const override { - auto const other_a (boost::polymorphic_downcast const *> (&base_a)); + auto const other_a (boost::polymorphic_downcast const *> (&base_a)); if (!current.first.data () && !other_a->current.first.data ()) { @@ -181,23 +182,23 @@ class rocksdb_iterator : public store_iterator_impl } void clear () { - current.first = nano::rocksdb_val{}; - current.second = nano::rocksdb_val{}; + current.first = nano::store::rocksdb::db_val{}; + current.second = nano::store::rocksdb::db_val{}; debug_assert (is_end_sentinal ()); } - nano::rocksdb_iterator & operator= (nano::rocksdb_iterator && other_a) + nano::store::rocksdb::iterator & operator= (nano::store::rocksdb::iterator && other_a) { cursor = std::move (other_a.cursor); current = other_a.current; return *this; } - nano::store_iterator_impl & operator= (nano::store_iterator_impl const &) = delete; + store::iterator_impl & operator= (store::iterator_impl const &) = delete; std::unique_ptr<::rocksdb::Iterator> cursor; - std::pair current; + std::pair current; private: - ::rocksdb::Transaction * tx (nano::transaction const & transaction_a) const + ::rocksdb::Transaction * tx (store::transaction const & transaction_a) const { return static_cast<::rocksdb::Transaction *> (transaction_a.get_handle ()); } diff --git a/nano/store/rocksdb/online_weight.cpp b/nano/store/rocksdb/online_weight.cpp new file mode 100644 index 0000000000..c6f7e1dacf --- /dev/null +++ b/nano/store/rocksdb/online_weight.cpp @@ -0,0 +1,45 @@ +#include +#include + +nano::store::rocksdb::online_weight::online_weight (nano::store::rocksdb::component & store_a) : + store{ store_a } +{ +} + +void nano::store::rocksdb::online_weight::put (store::write_transaction const & transaction, uint64_t time, nano::amount const & amount) +{ + auto status = store.put (transaction, tables::online_weight, time, amount); + store.release_assert_success (status); +} + +void nano::store::rocksdb::online_weight::del (store::write_transaction const & transaction, uint64_t time) +{ + auto status = store.del (transaction, tables::online_weight, time); + store.release_assert_success (status); +} + +nano::store::iterator nano::store::rocksdb::online_weight::begin (store::transaction const & transaction) const +{ + return store.make_iterator (transaction, tables::online_weight); +} + +nano::store::iterator nano::store::rocksdb::online_weight::rbegin (store::transaction const & transaction) const +{ + return store.make_iterator (transaction, tables::online_weight, false); +} + +nano::store::iterator nano::store::rocksdb::online_weight::end () const +{ + return store::iterator (nullptr); +} + +size_t nano::store::rocksdb::online_weight::count (store::transaction const & transaction) const +{ + return store.count (transaction, tables::online_weight); +} + +void nano::store::rocksdb::online_weight::clear (store::write_transaction const & transaction) +{ + auto status = store.drop (transaction, tables::online_weight); + store.release_assert_success (status); +} diff --git a/nano/store/rocksdb/online_weight.hpp b/nano/store/rocksdb/online_weight.hpp new file mode 100644 index 0000000000..68718f3e0b --- /dev/null +++ b/nano/store/rocksdb/online_weight.hpp @@ -0,0 +1,26 @@ +#pragma once + +#include + +namespace nano::store::rocksdb +{ +class component; +} +namespace nano::store::rocksdb +{ +class online_weight : public nano::store::online_weight +{ +private: + nano::store::rocksdb::component & store; + +public: + explicit online_weight (nano::store::rocksdb::component & store_a); + void put (store::write_transaction const & transaction_a, uint64_t time_a, nano::amount const & amount_a) override; + void del (store::write_transaction const & transaction_a, uint64_t time_a) override; + store::iterator begin (store::transaction const & transaction_a) const override; + store::iterator rbegin (store::transaction const & transaction_a) const override; + store::iterator end () const override; + size_t count (store::transaction const & transaction_a) const override; + void clear (store::write_transaction const & transaction_a) override; +}; +} // namespace nano::store::rocksdb diff --git a/nano/store/rocksdb/peer.cpp b/nano/store/rocksdb/peer.cpp new file mode 100644 index 0000000000..e837521605 --- /dev/null +++ b/nano/store/rocksdb/peer.cpp @@ -0,0 +1,43 @@ +#include +#include + +nano::store::rocksdb::peer::peer (nano::store::rocksdb::component & store) : + store{ store } {}; + +void nano::store::rocksdb::peer::put (store::write_transaction const & transaction, nano::endpoint_key const & endpoint) +{ + auto status = store.put (transaction, tables::peers, endpoint, nullptr); + store.release_assert_success (status); +} + +void nano::store::rocksdb::peer::del (store::write_transaction const & transaction, nano::endpoint_key const & endpoint) +{ + auto status = store.del (transaction, tables::peers, endpoint); + store.release_assert_success (status); +} + +bool nano::store::rocksdb::peer::exists (store::transaction const & transaction, nano::endpoint_key const & endpoint) const +{ + return store.exists (transaction, tables::peers, endpoint); +} + +size_t nano::store::rocksdb::peer::count (store::transaction const & transaction) const +{ + return store.count (transaction, tables::peers); +} + +void nano::store::rocksdb::peer::clear (store::write_transaction const & transaction) +{ + auto status = store.drop (transaction, tables::peers); + store.release_assert_success (status); +} + +nano::store::iterator nano::store::rocksdb::peer::begin (store::transaction const & transaction) const +{ + return store.make_iterator (transaction, tables::peers); +} + +nano::store::iterator nano::store::rocksdb::peer::end () const +{ + return store::iterator (nullptr); +} diff --git a/nano/store/rocksdb/peer.hpp b/nano/store/rocksdb/peer.hpp new file mode 100644 index 0000000000..3c6473701a --- /dev/null +++ b/nano/store/rocksdb/peer.hpp @@ -0,0 +1,26 @@ +#pragma once + +#include + +namespace nano::store::rocksdb +{ +class component; +} +namespace nano::store::rocksdb +{ +class peer : public nano::store::peer +{ +private: + nano::store::rocksdb::component & store; + +public: + explicit peer (nano::store::rocksdb::component & store_a); + void put (store::write_transaction const & transaction_a, nano::endpoint_key const & endpoint_a) override; + void del (store::write_transaction const & transaction_a, nano::endpoint_key const & endpoint_a) override; + bool exists (store::transaction const & transaction_a, nano::endpoint_key const & endpoint_a) const override; + size_t count (store::transaction const & transaction_a) const override; + void clear (store::write_transaction const & transaction_a) override; + store::iterator begin (store::transaction const & transaction_a) const override; + store::iterator end () const override; +}; +} // namespace nano::store::rocksdb diff --git a/nano/node/rocksdb/pending_store.cpp b/nano/store/rocksdb/pending.cpp similarity index 52% rename from nano/node/rocksdb/pending_store.cpp rename to nano/store/rocksdb/pending.cpp index b8f323582a..f7e843f184 100644 --- a/nano/node/rocksdb/pending_store.cpp +++ b/nano/store/rocksdb/pending.cpp @@ -1,25 +1,25 @@ -#include -#include #include +#include +#include -nano::rocksdb::pending_store::pending_store (nano::rocksdb::store & store) : +nano::store::rocksdb::pending::pending (nano::store::rocksdb::component & store) : store{ store } {}; -void nano::rocksdb::pending_store::put (nano::write_transaction const & transaction, nano::pending_key const & key, nano::pending_info const & pending) +void nano::store::rocksdb::pending::put (store::write_transaction const & transaction, nano::pending_key const & key, nano::pending_info const & pending) { auto status = store.put (transaction, tables::pending, key, pending); store.release_assert_success (status); } -void nano::rocksdb::pending_store::del (nano::write_transaction const & transaction, nano::pending_key const & key) +void nano::store::rocksdb::pending::del (store::write_transaction const & transaction, nano::pending_key const & key) { auto status = store.del (transaction, tables::pending, key); store.release_assert_success (status); } -bool nano::rocksdb::pending_store::get (nano::transaction const & transaction, nano::pending_key const & key, nano::pending_info & pending) +bool nano::store::rocksdb::pending::get (store::transaction const & transaction, nano::pending_key const & key, nano::pending_info & pending) { - nano::rocksdb_val value; + nano::store::rocksdb::db_val value; auto status1 = store.get (transaction, tables::pending, key, value); release_assert (store.success (status1) || store.not_found (status1)); bool result (true); @@ -31,34 +31,34 @@ bool nano::rocksdb::pending_store::get (nano::transaction const & transaction, n return result; } -bool nano::rocksdb::pending_store::exists (nano::transaction const & transaction_a, nano::pending_key const & key_a) +bool nano::store::rocksdb::pending::exists (store::transaction const & transaction_a, nano::pending_key const & key_a) { auto iterator (begin (transaction_a, key_a)); return iterator != end () && nano::pending_key (iterator->first) == key_a; } -bool nano::rocksdb::pending_store::any (nano::transaction const & transaction_a, nano::account const & account_a) +bool nano::store::rocksdb::pending::any (store::transaction const & transaction_a, nano::account const & account_a) { auto iterator (begin (transaction_a, nano::pending_key (account_a, 0))); return iterator != end () && nano::pending_key (iterator->first).account == account_a; } -nano::store_iterator nano::rocksdb::pending_store::begin (nano::transaction const & transaction_a, nano::pending_key const & key_a) const +nano::store::iterator nano::store::rocksdb::pending::begin (store::transaction const & transaction_a, nano::pending_key const & key_a) const { return store.template make_iterator (transaction_a, tables::pending, key_a); } -nano::store_iterator nano::rocksdb::pending_store::begin (nano::transaction const & transaction_a) const +nano::store::iterator nano::store::rocksdb::pending::begin (store::transaction const & transaction_a) const { return store.template make_iterator (transaction_a, tables::pending); } -nano::store_iterator nano::rocksdb::pending_store::end () const +nano::store::iterator nano::store::rocksdb::pending::end () const { - return nano::store_iterator (nullptr); + return store::iterator (nullptr); } -void nano::rocksdb::pending_store::for_each_par (std::function, nano::store_iterator)> const & action_a) const +void nano::store::rocksdb::pending::for_each_par (std::function, store::iterator)> const & action_a) const { parallel_traversal ( [&action_a, this] (nano::uint512_t const & start, nano::uint512_t const & end, bool const is_last) { diff --git a/nano/store/rocksdb/pending.hpp b/nano/store/rocksdb/pending.hpp new file mode 100644 index 0000000000..76a5f67d6d --- /dev/null +++ b/nano/store/rocksdb/pending.hpp @@ -0,0 +1,24 @@ +#pragma once + +#include + +namespace nano::store::rocksdb +{ +class pending : public nano::store::pending +{ +private: + nano::store::rocksdb::component & store; + +public: + explicit pending (nano::store::rocksdb::component & store_a); + void put (store::write_transaction const & transaction_a, nano::pending_key const & key_a, nano::pending_info const & pending_info_a) override; + void del (store::write_transaction const & transaction_a, nano::pending_key const & key_a) override; + bool get (store::transaction const & transaction_a, nano::pending_key const & key_a, nano::pending_info & pending_a) override; + bool exists (store::transaction const & transaction_a, nano::pending_key const & key_a) override; + bool any (store::transaction const & transaction_a, nano::account const & account_a) override; + store::iterator begin (store::transaction const & transaction_a, nano::pending_key const & key_a) const override; + store::iterator begin (store::transaction const & transaction_a) const override; + store::iterator end () const override; + void for_each_par (std::function, store::iterator)> const & action_a) const override; +}; +} // namespace nano::store::rocksdb diff --git a/nano/store/rocksdb/pruned.cpp b/nano/store/rocksdb/pruned.cpp new file mode 100644 index 0000000000..41d7fbb586 --- /dev/null +++ b/nano/store/rocksdb/pruned.cpp @@ -0,0 +1,70 @@ +#include +#include +#include + +nano::store::rocksdb::pruned::pruned (nano::store::rocksdb::component & store_a) : + store{ store_a } {}; + +void nano::store::rocksdb::pruned::put (store::write_transaction const & transaction_a, nano::block_hash const & hash_a) +{ + auto status = store.put (transaction_a, tables::pruned, hash_a, nullptr); + store.release_assert_success (status); +} + +void nano::store::rocksdb::pruned::del (store::write_transaction const & transaction_a, nano::block_hash const & hash_a) +{ + auto status = store.del (transaction_a, tables::pruned, hash_a); + store.release_assert_success (status); +} + +bool nano::store::rocksdb::pruned::exists (store::transaction const & transaction, nano::block_hash const & hash_a) const +{ + return store.exists (transaction, tables::pruned, hash_a); +} + +nano::block_hash nano::store::rocksdb::pruned::random (store::transaction const & transaction) +{ + nano::block_hash random_hash; + nano::random_pool::generate_block (random_hash.bytes.data (), random_hash.bytes.size ()); + auto existing = begin (transaction, random_hash); + if (existing == end ()) + { + existing = begin (transaction); + } + return existing != end () ? existing->first : 0; +} + +size_t nano::store::rocksdb::pruned::count (store::transaction const & transaction_a) const +{ + return store.count (transaction_a, tables::pruned); +} + +void nano::store::rocksdb::pruned::clear (store::write_transaction const & transaction_a) +{ + auto status = store.drop (transaction_a, tables::pruned); + store.release_assert_success (status); +} + +nano::store::iterator nano::store::rocksdb::pruned::begin (store::transaction const & transaction_a, nano::block_hash const & hash_a) const +{ + return store.make_iterator (transaction_a, tables::pruned, hash_a); +} + +nano::store::iterator nano::store::rocksdb::pruned::begin (store::transaction const & transaction_a) const +{ + return store.make_iterator (transaction_a, tables::pruned); +} + +nano::store::iterator nano::store::rocksdb::pruned::end () const +{ + return store::iterator (nullptr); +} + +void nano::store::rocksdb::pruned::for_each_par (std::function, store::iterator)> const & action_a) const +{ + parallel_traversal ( + [&action_a, this] (nano::uint256_t const & start, nano::uint256_t const & end, bool const is_last) { + auto transaction (this->store.tx_begin_read ()); + action_a (transaction, this->begin (transaction, start), !is_last ? this->begin (transaction, end) : this->end ()); + }); +} diff --git a/nano/store/rocksdb/pruned.hpp b/nano/store/rocksdb/pruned.hpp new file mode 100644 index 0000000000..aa5db1fbab --- /dev/null +++ b/nano/store/rocksdb/pruned.hpp @@ -0,0 +1,29 @@ +#pragma once + +#include + +namespace nano::store::rocksdb +{ +class component; +} +namespace nano::store::rocksdb +{ +class pruned : public nano::store::pruned +{ +private: + nano::store::rocksdb::component & store; + +public: + explicit pruned (nano::store::rocksdb::component & store_a); + void put (store::write_transaction const & transaction_a, nano::block_hash const & hash_a) override; + void del (store::write_transaction const & transaction_a, nano::block_hash const & hash_a) override; + bool exists (store::transaction const & transaction_a, nano::block_hash const & hash_a) const override; + nano::block_hash random (store::transaction const & transaction_a) override; + size_t count (store::transaction const & transaction_a) const override; + void clear (store::write_transaction const & transaction_a) override; + store::iterator begin (store::transaction const & transaction_a, nano::block_hash const & hash_a) const override; + store::iterator begin (store::transaction const & transaction_a) const override; + store::iterator end () const override; + void for_each_par (std::function, store::iterator)> const & action_a) const override; +}; +} // namespace nano::store::rocksdb diff --git a/nano/node/rocksdb/rocksdb.cpp b/nano/store/rocksdb/rocksdb.cpp similarity index 82% rename from nano/node/rocksdb/rocksdb.cpp rename to nano/store/rocksdb/rocksdb.cpp index 7beac09aa4..9b2aa7eb8a 100644 --- a/nano/node/rocksdb/rocksdb.cpp +++ b/nano/store/rocksdb/rocksdb.cpp @@ -1,8 +1,10 @@ #include -#include -#include -#include +#include +#include +#include +#include +#include #include #include #include @@ -33,36 +35,9 @@ class event_listener : public rocksdb::EventListener }; } -namespace nano -{ -template <> -void * rocksdb_val::data () const -{ - return (void *)value.data (); -} - -template <> -std::size_t rocksdb_val::size () const -{ - return value.size (); -} - -template <> -rocksdb_val::db_val (std::size_t size_a, void * data_a) : - value (static_cast (data_a), size_a) -{ -} - -template <> -void rocksdb_val::convert_buffer_to_value () -{ - value = ::rocksdb::Slice (reinterpret_cast (buffer->data ()), buffer->size ()); -} -} - -nano::rocksdb::store::store (nano::logger_mt & logger_a, boost::filesystem::path const & path_a, nano::ledger_constants & constants, nano::rocksdb_config const & rocksdb_config_a, bool open_read_only_a) : +nano::store::rocksdb::component::component (nano::logger_mt & logger_a, boost::filesystem::path const & path_a, nano::ledger_constants & constants, nano::rocksdb_config const & rocksdb_config_a, bool open_read_only_a) : // clang-format off - nano::store{ + nano::store::component{ block_store, frontier_store, account_store, @@ -181,7 +156,7 @@ nano::rocksdb::store::store (nano::logger_mt & logger_a, boost::filesystem::path } } -std::unordered_map nano::rocksdb::store::create_cf_name_table_map () const +std::unordered_map nano::store::rocksdb::component::create_cf_name_table_map () const { std::unordered_map map{ { ::rocksdb::kDefaultColumnFamilyName.c_str (), tables::default_unused }, { "frontiers", tables::frontiers }, @@ -200,7 +175,7 @@ std::unordered_map nano::rocksdb::store::create_cf_n return map; } -void nano::rocksdb::store::open (bool & error_a, boost::filesystem::path const & path_a, bool open_read_only_a, ::rocksdb::Options const & options_a, std::vector<::rocksdb::ColumnFamilyDescriptor> column_families) +void nano::store::rocksdb::component::open (bool & error_a, boost::filesystem::path const & path_a, bool open_read_only_a, ::rocksdb::Options const & options_a, std::vector<::rocksdb::ColumnFamilyDescriptor> column_families) { // auto options = get_db_options (); ::rocksdb::Status s; @@ -231,7 +206,7 @@ void nano::rocksdb::store::open (bool & error_a, boost::filesystem::path const & error_a |= !s.ok (); } -bool nano::rocksdb::store::do_upgrades (nano::write_transaction const & transaction_a) +bool nano::store::rocksdb::component::do_upgrades (store::write_transaction const & transaction_a) { bool error_l{ false }; auto version_l = version.get (transaction_a); @@ -273,7 +248,7 @@ bool nano::rocksdb::store::do_upgrades (nano::write_transaction const & transact return error_l; } -void nano::rocksdb::store::upgrade_v21_to_v22 (nano::write_transaction const & transaction_a) +void nano::store::rocksdb::component::upgrade_v21_to_v22 (store::write_transaction const & transaction_a) { logger.always_log ("Preparing v21 to v22 database upgrade..."); if (column_family_exists ("unchecked")) @@ -295,14 +270,14 @@ void nano::rocksdb::store::upgrade_v21_to_v22 (nano::write_transaction const & t logger.always_log ("Finished removing unchecked table"); } -void nano::rocksdb::store::generate_tombstone_map () +void nano::store::rocksdb::component::generate_tombstone_map () { tombstone_map.emplace (std::piecewise_construct, std::forward_as_tuple (nano::tables::blocks), std::forward_as_tuple (0, 25000)); tombstone_map.emplace (std::piecewise_construct, std::forward_as_tuple (nano::tables::accounts), std::forward_as_tuple (0, 25000)); tombstone_map.emplace (std::piecewise_construct, std::forward_as_tuple (nano::tables::pending), std::forward_as_tuple (0, 25000)); } -rocksdb::ColumnFamilyOptions nano::rocksdb::store::get_common_cf_options (std::shared_ptr<::rocksdb::TableFactory> const & table_factory_a, unsigned long long memtable_size_bytes_a) const +rocksdb::ColumnFamilyOptions nano::store::rocksdb::component::get_common_cf_options (std::shared_ptr<::rocksdb::TableFactory> const & table_factory_a, unsigned long long memtable_size_bytes_a) const { ::rocksdb::ColumnFamilyOptions cf_options; cf_options.table_factory = table_factory_a; @@ -334,7 +309,7 @@ rocksdb::ColumnFamilyOptions nano::rocksdb::store::get_common_cf_options (std::s return cf_options; } -rocksdb::ColumnFamilyOptions nano::rocksdb::store::get_cf_options (std::string const & cf_name_a) const +rocksdb::ColumnFamilyOptions nano::store::rocksdb::component::get_cf_options (std::string const & cf_name_a) const { ::rocksdb::ColumnFamilyOptions cf_options; auto const memtable_size_bytes = base_memtable_size_bytes (); @@ -414,7 +389,7 @@ rocksdb::ColumnFamilyOptions nano::rocksdb::store::get_cf_options (std::string c return cf_options; } -std::vector nano::rocksdb::store::create_column_families () +std::vector nano::store::rocksdb::component::create_column_families () { std::vector<::rocksdb::ColumnFamilyDescriptor> column_families; for (auto & [cf_name, table] : cf_name_table_map) @@ -425,37 +400,37 @@ std::vector nano::rocksdb::store::create_column return column_families; } -nano::write_transaction nano::rocksdb::store::tx_begin_write (std::vector const & tables_requiring_locks_a, std::vector const & tables_no_locks_a) +nano::store::write_transaction nano::store::rocksdb::component::tx_begin_write (std::vector const & tables_requiring_locks_a, std::vector const & tables_no_locks_a) { - std::unique_ptr txn; + std::unique_ptr txn; release_assert (optimistic_db != nullptr); if (tables_requiring_locks_a.empty () && tables_no_locks_a.empty ()) { // Use all tables if none are specified - txn = std::make_unique (optimistic_db, all_tables (), tables_no_locks_a, write_lock_mutexes); + txn = std::make_unique (optimistic_db, all_tables (), tables_no_locks_a, write_lock_mutexes); } else { - txn = std::make_unique (optimistic_db, tables_requiring_locks_a, tables_no_locks_a, write_lock_mutexes); + txn = std::make_unique (optimistic_db, tables_requiring_locks_a, tables_no_locks_a, write_lock_mutexes); } // Tables must be kept in alphabetical order. These can be used for mutex locking, so order is important to prevent deadlocking debug_assert (std::is_sorted (tables_requiring_locks_a.begin (), tables_requiring_locks_a.end ())); - return nano::write_transaction{ std::move (txn) }; + return store::write_transaction{ std::move (txn) }; } -nano::read_transaction nano::rocksdb::store::tx_begin_read () const +nano::store::read_transaction nano::store::rocksdb::component::tx_begin_read () const { - return nano::read_transaction{ std::make_unique (db.get ()) }; + return store::read_transaction{ std::make_unique (db.get ()) }; } -std::string nano::rocksdb::store::vendor_get () const +std::string nano::store::rocksdb::component::vendor_get () const { return boost::str (boost::format ("RocksDB %1%.%2%.%3%") % ROCKSDB_MAJOR % ROCKSDB_MINOR % ROCKSDB_PATCH); } -std::vector<::rocksdb::ColumnFamilyDescriptor> nano::rocksdb::store::get_single_column_family (std::string cf_name) const +std::vector<::rocksdb::ColumnFamilyDescriptor> nano::store::rocksdb::component::get_single_column_family (std::string cf_name) const { std::vector<::rocksdb::ColumnFamilyDescriptor> minimum_cf_set{ { ::rocksdb::kDefaultColumnFamilyName, ::rocksdb::ColumnFamilyOptions{} }, @@ -464,7 +439,7 @@ std::vector<::rocksdb::ColumnFamilyDescriptor> nano::rocksdb::store::get_single_ return minimum_cf_set; } -std::vector<::rocksdb::ColumnFamilyDescriptor> nano::rocksdb::store::get_current_column_families (std::string const & path_a, ::rocksdb::Options const & options_a) const +std::vector<::rocksdb::ColumnFamilyDescriptor> nano::store::rocksdb::component::get_current_column_families (std::string const & path_a, ::rocksdb::Options const & options_a) const { std::vector<::rocksdb::ColumnFamilyDescriptor> column_families; @@ -482,7 +457,7 @@ std::vector<::rocksdb::ColumnFamilyDescriptor> nano::rocksdb::store::get_current return column_families; } -rocksdb::ColumnFamilyHandle * nano::rocksdb::store::get_column_family (char const * name) const +rocksdb::ColumnFamilyHandle * nano::store::rocksdb::component::get_column_family (char const * name) const { auto & handles_l = handles; auto iter = std::find_if (handles_l.begin (), handles_l.end (), [name] (auto & handle) { @@ -492,7 +467,7 @@ rocksdb::ColumnFamilyHandle * nano::rocksdb::store::get_column_family (char cons return (*iter).get (); } -bool nano::rocksdb::store::column_family_exists (char const * name) const +bool nano::store::rocksdb::component::column_family_exists (char const * name) const { auto & handles_l = handles; auto iter = std::find_if (handles_l.begin (), handles_l.end (), [name] (auto & handle) { @@ -501,7 +476,7 @@ bool nano::rocksdb::store::column_family_exists (char const * name) const return (iter != handles_l.end ()); } -rocksdb::ColumnFamilyHandle * nano::rocksdb::store::table_to_column_family (tables table_a) const +rocksdb::ColumnFamilyHandle * nano::store::rocksdb::component::table_to_column_family (tables table_a) const { switch (table_a) { @@ -533,7 +508,7 @@ rocksdb::ColumnFamilyHandle * nano::rocksdb::store::table_to_column_family (tabl } } -bool nano::rocksdb::store::exists (nano::transaction const & transaction_a, tables table_a, nano::rocksdb_val const & key_a) const +bool nano::store::rocksdb::component::exists (store::transaction const & transaction_a, tables table_a, nano::store::rocksdb::db_val const & key_a) const { ::rocksdb::PinnableSlice slice; ::rocksdb::Status status; @@ -551,7 +526,7 @@ bool nano::rocksdb::store::exists (nano::transaction const & transaction_a, tabl return (status.ok ()); } -int nano::rocksdb::store::del (nano::write_transaction const & transaction_a, tables table_a, nano::rocksdb_val const & key_a) +int nano::store::rocksdb::component::del (store::write_transaction const & transaction_a, tables table_a, nano::store::rocksdb::db_val const & key_a) { debug_assert (transaction_a.contains (table_a)); // RocksDB does not report not_found status, it is a pre-condition that the key exists @@ -560,7 +535,7 @@ int nano::rocksdb::store::del (nano::write_transaction const & transaction_a, ta return tx (transaction_a)->Delete (table_to_column_family (table_a), key_a).code (); } -void nano::rocksdb::store::flush_tombstones_check (tables table_a) +void nano::store::rocksdb::component::flush_tombstones_check (tables table_a) { // Update the number of deletes for some tables, and force a flush if there are too many tombstones // as it can affect read performance. @@ -575,18 +550,18 @@ void nano::rocksdb::store::flush_tombstones_check (tables table_a) } } -void nano::rocksdb::store::flush_table (nano::tables table_a) +void nano::store::rocksdb::component::flush_table (nano::tables table_a) { db->Flush (::rocksdb::FlushOptions{}, table_to_column_family (table_a)); } -rocksdb::Transaction * nano::rocksdb::store::tx (nano::transaction const & transaction_a) const +rocksdb::Transaction * nano::store::rocksdb::component::tx (store::transaction const & transaction_a) const { debug_assert (!is_read (transaction_a)); return static_cast<::rocksdb::Transaction *> (transaction_a.get_handle ()); } -int nano::rocksdb::store::get (nano::transaction const & transaction_a, tables table_a, nano::rocksdb_val const & key_a, nano::rocksdb_val & value_a) const +int nano::store::rocksdb::component::get (store::transaction const & transaction_a, tables table_a, nano::store::rocksdb::db_val const & key_a, nano::store::rocksdb::db_val & value_a) const { ::rocksdb::ReadOptions options; ::rocksdb::PinnableSlice slice; @@ -610,29 +585,29 @@ int nano::rocksdb::store::get (nano::transaction const & transaction_a, tables t return status.code (); } -int nano::rocksdb::store::put (nano::write_transaction const & transaction_a, tables table_a, nano::rocksdb_val const & key_a, nano::rocksdb_val const & value_a) +int nano::store::rocksdb::component::put (store::write_transaction const & transaction_a, tables table_a, nano::store::rocksdb::db_val const & key_a, nano::store::rocksdb::db_val const & value_a) { debug_assert (transaction_a.contains (table_a)); auto txn = tx (transaction_a); return txn->Put (table_to_column_family (table_a), key_a, value_a).code (); } -bool nano::rocksdb::store::not_found (int status) const +bool nano::store::rocksdb::component::not_found (int status) const { return (status_code_not_found () == status); } -bool nano::rocksdb::store::success (int status) const +bool nano::store::rocksdb::component::success (int status) const { return (static_cast (::rocksdb::Status::Code::kOk) == status); } -int nano::rocksdb::store::status_code_not_found () const +int nano::store::rocksdb::component::status_code_not_found () const { return static_cast (::rocksdb::Status::Code::kNotFound); } -uint64_t nano::rocksdb::store::count (nano::transaction const & transaction_a, tables table_a) const +uint64_t nano::store::rocksdb::component::count (store::transaction const & transaction_a, tables table_a) const { uint64_t sum = 0; // Peers/online weight are small enough that they can just be iterated to get accurate counts. @@ -693,7 +668,7 @@ uint64_t nano::rocksdb::store::count (nano::transaction const & transaction_a, t return sum; } -int nano::rocksdb::store::drop (nano::write_transaction const & transaction_a, tables table_a) +int nano::store::rocksdb::component::drop (store::write_transaction const & transaction_a, tables table_a) { debug_assert (transaction_a.contains (table_a)); auto col = table_to_column_family (table_a); @@ -707,7 +682,7 @@ int nano::rocksdb::store::drop (nano::write_transaction const & transaction_a, t int status = 0; for (auto i = peer.begin (transaction_a), n = peer.end (); i != n; ++i) { - status = del (transaction_a, tables::peers, nano::rocksdb_val (i->first)); + status = del (transaction_a, tables::peers, nano::store::rocksdb::db_val (i->first)); release_assert (success (status)); } return status; @@ -720,7 +695,7 @@ int nano::rocksdb::store::drop (nano::write_transaction const & transaction_a, t return status; } -int nano::rocksdb::store::clear (::rocksdb::ColumnFamilyHandle * column_family) +int nano::store::rocksdb::component::clear (::rocksdb::ColumnFamilyHandle * column_family) { // Dropping completely removes the column auto name = column_family->GetName (); @@ -738,7 +713,7 @@ int nano::rocksdb::store::clear (::rocksdb::ColumnFamilyHandle * column_family) return status.code (); } -void nano::rocksdb::store::construct_column_family_mutexes () +void nano::store::rocksdb::component::construct_column_family_mutexes () { for (auto table : all_tables ()) { @@ -746,7 +721,7 @@ void nano::rocksdb::store::construct_column_family_mutexes () } } -rocksdb::Options nano::rocksdb::store::get_db_options () +rocksdb::Options nano::store::rocksdb::component::get_db_options () { ::rocksdb::Options db_options; db_options.create_if_missing = true; @@ -788,7 +763,7 @@ rocksdb::Options nano::rocksdb::store::get_db_options () return db_options; } -rocksdb::BlockBasedTableOptions nano::rocksdb::store::get_active_table_options (std::size_t lru_size) const +rocksdb::BlockBasedTableOptions nano::store::rocksdb::component::get_active_table_options (std::size_t lru_size) const { ::rocksdb::BlockBasedTableOptions table_options; @@ -817,7 +792,7 @@ rocksdb::BlockBasedTableOptions nano::rocksdb::store::get_active_table_options ( return table_options; } -rocksdb::BlockBasedTableOptions nano::rocksdb::store::get_small_table_options () const +rocksdb::BlockBasedTableOptions nano::store::rocksdb::component::get_small_table_options () const { ::rocksdb::BlockBasedTableOptions table_options; // Improve point lookup performance be using the data block hash index (uses about 5% more space). @@ -827,7 +802,7 @@ rocksdb::BlockBasedTableOptions nano::rocksdb::store::get_small_table_options () return table_options; } -rocksdb::ColumnFamilyOptions nano::rocksdb::store::get_small_cf_options (std::shared_ptr<::rocksdb::TableFactory> const & table_factory_a) const +rocksdb::ColumnFamilyOptions nano::store::rocksdb::component::get_small_cf_options (std::shared_ptr<::rocksdb::TableFactory> const & table_factory_a) const { auto const memtable_size_bytes = 10000; auto cf_options = get_common_cf_options (table_factory_a, memtable_size_bytes); @@ -841,7 +816,7 @@ rocksdb::ColumnFamilyOptions nano::rocksdb::store::get_small_cf_options (std::sh return cf_options; } -::rocksdb::ColumnFamilyOptions nano::rocksdb::store::get_active_cf_options (std::shared_ptr<::rocksdb::TableFactory> const & table_factory_a, unsigned long long memtable_size_bytes_a) const +::rocksdb::ColumnFamilyOptions nano::store::rocksdb::component::get_active_cf_options (std::shared_ptr<::rocksdb::TableFactory> const & table_factory_a, unsigned long long memtable_size_bytes_a) const { auto cf_options = get_common_cf_options (table_factory_a, memtable_size_bytes_a); @@ -857,7 +832,7 @@ ::rocksdb::ColumnFamilyOptions nano::rocksdb::store::get_active_cf_options (std: return cf_options; } -void nano::rocksdb::store::on_flush (::rocksdb::FlushJobInfo const & flush_job_info_a) +void nano::store::rocksdb::component::on_flush (::rocksdb::FlushJobInfo const & flush_job_info_a) { // Reset appropriate tombstone counters if (auto it = tombstone_map.find (cf_name_table_map[flush_job_info_a.cf_name.c_str ()]); it != tombstone_map.end ()) @@ -866,12 +841,12 @@ void nano::rocksdb::store::on_flush (::rocksdb::FlushJobInfo const & flush_job_i } } -std::vector nano::rocksdb::store::all_tables () const +std::vector nano::store::rocksdb::component::all_tables () const { return std::vector{ tables::accounts, tables::blocks, tables::confirmation_height, tables::final_votes, tables::frontiers, tables::meta, tables::online_weight, tables::peers, tables::pending, tables::pruned, tables::vote }; } -bool nano::rocksdb::store::copy_db (boost::filesystem::path const & destination_path) +bool nano::store::rocksdb::component::copy_db (boost::filesystem::path const & destination_path) { std::unique_ptr<::rocksdb::BackupEngine> backup_engine; { @@ -935,23 +910,23 @@ bool nano::rocksdb::store::copy_db (boost::filesystem::path const & destination_ // Open it so that it flushes all WAL files if (status.ok ()) { - nano::rocksdb::store rocksdb_store{ logger, destination_path.string (), constants, rocksdb_config, false }; + nano::store::rocksdb::component rocksdb_store{ logger, destination_path.string (), constants, rocksdb_config, false }; return !rocksdb_store.init_error (); } return false; } -void nano::rocksdb::store::rebuild_db (nano::write_transaction const & transaction_a) +void nano::store::rocksdb::component::rebuild_db (store::write_transaction const & transaction_a) { // Not available for RocksDB } -bool nano::rocksdb::store::init_error () const +bool nano::store::rocksdb::component::init_error () const { return error; } -void nano::rocksdb::store::serialize_memory_stats (boost::property_tree::ptree & json) +void nano::store::rocksdb::component::serialize_memory_stats (boost::property_tree::ptree & json) { uint64_t val; @@ -998,28 +973,28 @@ void nano::rocksdb::store::serialize_memory_stats (boost::property_tree::ptree & json.put ("block-cache-usage", val); } -unsigned long long nano::rocksdb::store::blocks_memtable_size_bytes () const +unsigned long long nano::store::rocksdb::component::blocks_memtable_size_bytes () const { return base_memtable_size_bytes (); } -unsigned long long nano::rocksdb::store::base_memtable_size_bytes () const +unsigned long long nano::store::rocksdb::component::base_memtable_size_bytes () const { return 1024ULL * 1024 * rocksdb_config.memory_multiplier * base_memtable_size; } // This is a ratio of the blocks memtable size to keep total write transaction commit size down. -unsigned nano::rocksdb::store::max_block_write_batch_num () const +unsigned nano::store::rocksdb::component::max_block_write_batch_num () const { return max_block_write_batch_num_m; } -std::string nano::rocksdb::store::error_string (int status) const +std::string nano::store::rocksdb::component::error_string (int status) const { return std::to_string (status); } -nano::rocksdb::store::tombstone_info::tombstone_info (uint64_t num_since_last_flush_a, uint64_t const max_a) : +nano::store::rocksdb::component::tombstone_info::tombstone_info (uint64_t num_since_last_flush_a, uint64_t const max_a) : num_since_last_flush (num_since_last_flush_a), max (max_a) { diff --git a/nano/store/rocksdb/rocksdb.hpp b/nano/store/rocksdb/rocksdb.hpp new file mode 100644 index 0000000000..59484937e2 --- /dev/null +++ b/nano/store/rocksdb/rocksdb.hpp @@ -0,0 +1,180 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace nano +{ +class logging_mt; +class rocksdb_config; +class rocksdb_block_store_tombstone_count_Test; +} + +namespace nano::store::rocksdb +{ +class rocksdb_block_store_upgrade_v21_v22_Test; + +/** + * rocksdb implementation of the block store + */ +class component : public nano::store::component +{ +private: + nano::store::rocksdb::account account_store; + nano::store::rocksdb::block block_store; + nano::store::rocksdb::confirmation_height confirmation_height_store; + nano::store::rocksdb::final_vote final_vote_store; + nano::store::rocksdb::frontier frontier_store; + nano::store::rocksdb::online_weight online_weight_store; + nano::store::rocksdb::peer peer_store; + nano::store::rocksdb::pending pending_store; + nano::store::rocksdb::pruned pruned_store; + nano::store::rocksdb::version version_store; + +public: + friend class nano::store::rocksdb::account; + friend class nano::store::rocksdb::block; + friend class nano::store::rocksdb::confirmation_height; + friend class nano::store::rocksdb::final_vote; + friend class nano::store::rocksdb::frontier; + friend class nano::store::rocksdb::online_weight; + friend class nano::store::rocksdb::peer; + friend class nano::store::rocksdb::pending; + friend class nano::store::rocksdb::pruned; + friend class nano::store::rocksdb::version; + + explicit component (nano::logger_mt &, boost::filesystem::path const &, nano::ledger_constants & constants, nano::rocksdb_config const & = nano::rocksdb_config{}, bool open_read_only = false); + + store::write_transaction tx_begin_write (std::vector const & tables_requiring_lock = {}, std::vector const & tables_no_lock = {}) override; + store::read_transaction tx_begin_read () const override; + + std::string vendor_get () const override; + + uint64_t count (store::transaction const & transaction_a, tables table_a) const override; + + bool exists (store::transaction const & transaction_a, tables table_a, nano::store::rocksdb::db_val const & key_a) const; + int get (store::transaction const & transaction_a, tables table_a, nano::store::rocksdb::db_val const & key_a, nano::store::rocksdb::db_val & value_a) const; + int put (store::write_transaction const & transaction_a, tables table_a, nano::store::rocksdb::db_val const & key_a, nano::store::rocksdb::db_val const & value_a); + int del (store::write_transaction const & transaction_a, tables table_a, nano::store::rocksdb::db_val const & key_a); + + void serialize_memory_stats (boost::property_tree::ptree &) override; + + bool copy_db (boost::filesystem::path const & destination) override; + void rebuild_db (store::write_transaction const & transaction_a) override; + + unsigned max_block_write_batch_num () const override; + + template + store::iterator make_iterator (store::transaction const & transaction_a, tables table_a, bool const direction_asc = true) const + { + return store::iterator (std::make_unique> (db.get (), transaction_a, table_to_column_family (table_a), nullptr, direction_asc)); + } + + template + store::iterator make_iterator (store::transaction const & transaction_a, tables table_a, nano::store::rocksdb::db_val const & key) const + { + return store::iterator (std::make_unique> (db.get (), transaction_a, table_to_column_family (table_a), &key, true)); + } + + bool init_error () const override; + + std::string error_string (int status) const override; + +private: + bool error{ false }; + nano::logger_mt & logger; + nano::ledger_constants & constants; + // Optimistic transactions are used in write mode + ::rocksdb::OptimisticTransactionDB * optimistic_db = nullptr; + std::unique_ptr<::rocksdb::DB> db; + std::vector> handles; + std::shared_ptr<::rocksdb::TableFactory> small_table_factory; + std::unordered_map write_lock_mutexes; + nano::rocksdb_config rocksdb_config; + unsigned const max_block_write_batch_num_m; + + class tombstone_info + { + public: + tombstone_info (uint64_t, uint64_t const); + std::atomic num_since_last_flush; + uint64_t const max; + }; + + std::unordered_map tombstone_map; + std::unordered_map cf_name_table_map; + + ::rocksdb::Transaction * tx (store::transaction const & transaction_a) const; + std::vector all_tables () const; + + bool not_found (int status) const override; + bool success (int status) const override; + void release_assert_success (int const status) const + { + if (!success (status)) + { + release_assert (false, error_string (status)); + } + } + int status_code_not_found () const override; + int drop (store::write_transaction const &, tables) override; + + std::vector<::rocksdb::ColumnFamilyDescriptor> get_single_column_family (std::string cf_name) const; + std::vector<::rocksdb::ColumnFamilyDescriptor> get_current_column_families (std::string const & path_a, ::rocksdb::Options const & options_a) const; + ::rocksdb::ColumnFamilyHandle * get_column_family (char const * name) const; + bool column_family_exists (char const * name) const; + ::rocksdb::ColumnFamilyHandle * table_to_column_family (tables table_a) const; + int clear (::rocksdb::ColumnFamilyHandle * column_family); + + void open (bool & error_a, boost::filesystem::path const & path_a, bool open_read_only_a, ::rocksdb::Options const & options_a, std::vector<::rocksdb::ColumnFamilyDescriptor> column_families); + + bool do_upgrades (store::write_transaction const &); + void upgrade_v21_to_v22 (store::write_transaction const &); + + void construct_column_family_mutexes (); + ::rocksdb::Options get_db_options (); + ::rocksdb::ColumnFamilyOptions get_common_cf_options (std::shared_ptr<::rocksdb::TableFactory> const & table_factory_a, unsigned long long memtable_size_bytes_a) const; + ::rocksdb::ColumnFamilyOptions get_active_cf_options (std::shared_ptr<::rocksdb::TableFactory> const & table_factory_a, unsigned long long memtable_size_bytes_a) const; + ::rocksdb::ColumnFamilyOptions get_small_cf_options (std::shared_ptr<::rocksdb::TableFactory> const & table_factory_a) const; + ::rocksdb::BlockBasedTableOptions get_active_table_options (std::size_t lru_size) const; + ::rocksdb::BlockBasedTableOptions get_small_table_options () const; + ::rocksdb::ColumnFamilyOptions get_cf_options (std::string const & cf_name_a) const; + + void on_flush (::rocksdb::FlushJobInfo const &); + void flush_table (nano::tables table_a); + void flush_tombstones_check (nano::tables table_a); + void generate_tombstone_map (); + std::unordered_map create_cf_name_table_map () const; + + std::vector<::rocksdb::ColumnFamilyDescriptor> create_column_families (); + unsigned long long base_memtable_size_bytes () const; + unsigned long long blocks_memtable_size_bytes () const; + + constexpr static int base_memtable_size = 16; + constexpr static int base_block_cache_size = 8; + + friend class nano::rocksdb_block_store_tombstone_count_Test; + friend class rocksdb_block_store_upgrade_v21_v22_Test; +}; +} // namespace nano::store::rocksdb diff --git a/nano/store/rocksdb/transaction.cpp b/nano/store/rocksdb/transaction.cpp new file mode 100644 index 0000000000..0974eeca62 --- /dev/null +++ b/nano/store/rocksdb/transaction.cpp @@ -0,0 +1,109 @@ +#include + +nano::store::rocksdb::read_transaction_impl::read_transaction_impl (::rocksdb::DB * db_a) : + db (db_a) +{ + if (db_a) + { + options.snapshot = db_a->GetSnapshot (); + } +} + +nano::store::rocksdb::read_transaction_impl::~read_transaction_impl () +{ + reset (); +} + +void nano::store::rocksdb::read_transaction_impl::reset () +{ + if (db) + { + db->ReleaseSnapshot (options.snapshot); + } +} + +void nano::store::rocksdb::read_transaction_impl::renew () +{ + options.snapshot = db->GetSnapshot (); +} + +void * nano::store::rocksdb::read_transaction_impl::get_handle () const +{ + return (void *)&options; +} + +nano::store::rocksdb::write_transaction_impl::write_transaction_impl (::rocksdb::OptimisticTransactionDB * db_a, std::vector const & tables_requiring_locks_a, std::vector const & tables_no_locks_a, std::unordered_map & mutexes_a) : + db (db_a), + tables_requiring_locks (tables_requiring_locks_a), + tables_no_locks (tables_no_locks_a), + mutexes (mutexes_a) +{ + lock (); + ::rocksdb::OptimisticTransactionOptions txn_options; + txn_options.set_snapshot = true; + txn = db->BeginTransaction (::rocksdb::WriteOptions (), txn_options); +} + +nano::store::rocksdb::write_transaction_impl::~write_transaction_impl () +{ + commit (); + delete txn; + unlock (); +} + +void nano::store::rocksdb::write_transaction_impl::commit () +{ + if (active) + { + auto status = txn->Commit (); + + // If there are no available memtables try again a few more times + constexpr auto num_attempts = 10; + auto attempt_num = 0; + while (status.IsTryAgain () && attempt_num < num_attempts) + { + status = txn->Commit (); + ++attempt_num; + } + + if (!status.ok ()) + { + release_assert (false && "Unable to write to the RocksDB database", status.ToString ()); + } + active = false; + } +} + +void nano::store::rocksdb::write_transaction_impl::renew () +{ + ::rocksdb::OptimisticTransactionOptions txn_options; + txn_options.set_snapshot = true; + db->BeginTransaction (::rocksdb::WriteOptions (), txn_options, txn); + active = true; +} + +void * nano::store::rocksdb::write_transaction_impl::get_handle () const +{ + return txn; +} + +void nano::store::rocksdb::write_transaction_impl::lock () +{ + for (auto table : tables_requiring_locks) + { + mutexes.at (table).lock (); + } +} + +void nano::store::rocksdb::write_transaction_impl::unlock () +{ + for (auto table : tables_requiring_locks) + { + mutexes.at (table).unlock (); + } +} + +bool nano::store::rocksdb::write_transaction_impl::contains (nano::tables table_a) const +{ + return (std::find (tables_requiring_locks.begin (), tables_requiring_locks.end (), table_a) != tables_requiring_locks.end ()) || (std::find (tables_no_locks.begin (), tables_no_locks.end (), table_a) != tables_no_locks.end ()); +} diff --git a/nano/node/rocksdb/rocksdb_txn.hpp b/nano/store/rocksdb/transaction_impl.hpp similarity index 58% rename from nano/node/rocksdb/rocksdb_txn.hpp rename to nano/store/rocksdb/transaction_impl.hpp index f77463da87..ce3f531f94 100644 --- a/nano/node/rocksdb/rocksdb_txn.hpp +++ b/nano/store/rocksdb/transaction_impl.hpp @@ -1,19 +1,19 @@ #pragma once -#include +#include #include #include #include #include -namespace nano +namespace nano::store::rocksdb { -class read_rocksdb_txn final : public read_transaction_impl +class read_transaction_impl final : public store::read_transaction_impl { public: - read_rocksdb_txn (::rocksdb::DB * db); - ~read_rocksdb_txn (); + read_transaction_impl (::rocksdb::DB * db); + ~read_transaction_impl (); void reset () override; void renew () override; void * get_handle () const override; @@ -23,11 +23,11 @@ class read_rocksdb_txn final : public read_transaction_impl ::rocksdb::ReadOptions options; }; -class write_rocksdb_txn final : public write_transaction_impl +class write_transaction_impl final : public store::write_transaction_impl { public: - write_rocksdb_txn (::rocksdb::OptimisticTransactionDB * db_a, std::vector const & tables_requiring_locks_a, std::vector const & tables_no_locks_a, std::unordered_map & mutexes_a); - ~write_rocksdb_txn (); + write_transaction_impl (::rocksdb::OptimisticTransactionDB * db_a, std::vector const & tables_requiring_locks_a, std::vector const & tables_no_locks_a, std::unordered_map & mutexes_a); + ~write_transaction_impl (); void commit () override; void renew () override; void * get_handle () const override; diff --git a/nano/node/rocksdb/version_store.cpp b/nano/store/rocksdb/version.cpp similarity index 62% rename from nano/node/rocksdb/version_store.cpp rename to nano/store/rocksdb/version.cpp index 04ea7c0036..c1df2e0232 100644 --- a/nano/node/rocksdb/version_store.cpp +++ b/nano/store/rocksdb/version.cpp @@ -1,10 +1,10 @@ -#include -#include +#include +#include -nano::rocksdb::version_store::version_store (nano::rocksdb::store & store_a) : +nano::store::rocksdb::version::version (nano::store::rocksdb::component & store_a) : store{ store_a } {}; -void nano::rocksdb::version_store::put (nano::write_transaction const & transaction_a, int version) +void nano::store::rocksdb::version::put (store::write_transaction const & transaction_a, int version) { nano::uint256_union version_key{ 1 }; nano::uint256_union version_value (version); @@ -12,10 +12,10 @@ void nano::rocksdb::version_store::put (nano::write_transaction const & transact store.release_assert_success (status); } -int nano::rocksdb::version_store::get (nano::transaction const & transaction_a) const +int nano::store::rocksdb::version::get (store::transaction const & transaction_a) const { nano::uint256_union version_key{ 1 }; - nano::rocksdb_val data; + nano::store::rocksdb::db_val data; auto status = store.get (transaction_a, tables::meta, version_key, data); int result = store.version_minimum; if (store.success (status)) diff --git a/nano/store/rocksdb/version.hpp b/nano/store/rocksdb/version.hpp new file mode 100644 index 0000000000..a5250b241c --- /dev/null +++ b/nano/store/rocksdb/version.hpp @@ -0,0 +1,17 @@ +#pragma once + +#include + +namespace nano::store::rocksdb +{ +class version : public nano::store::version +{ +protected: + nano::store::rocksdb::component & store; + +public: + explicit version (nano::store::rocksdb::component & store_a); + void put (store::write_transaction const & transaction_a, int version_a) override; + int get (store::transaction const & transaction_a) const override; +}; +} // namespace nano::store::rocksdb diff --git a/nano/store/tables.hpp b/nano/store/tables.hpp new file mode 100644 index 0000000000..e0a03bd020 --- /dev/null +++ b/nano/store/tables.hpp @@ -0,0 +1,35 @@ +#pragma once + +#include + +namespace nano +{ +// Keep this in alphabetical order +enum class tables +{ + accounts, + blocks, + confirmation_height, + default_unused, // RocksDB only + final_votes, + frontiers, + meta, + online_weight, + peers, + pending, + pruned, + vote +}; +} // namespace nano + +namespace std +{ +template <> +struct hash<::nano::tables> +{ + size_t operator() (::nano::tables const & table_a) const + { + return static_cast (table_a); + } +}; // struct hash +} // namespace std diff --git a/nano/store/transaction.cpp b/nano/store/transaction.cpp new file mode 100644 index 0000000000..7d30d403a0 --- /dev/null +++ b/nano/store/transaction.cpp @@ -0,0 +1,64 @@ +#include +#include +#include + +nano::store::read_transaction::read_transaction (std::unique_ptr read_transaction_impl) : + impl (std::move (read_transaction_impl)) +{ +} + +void * nano::store::read_transaction::get_handle () const +{ + return impl->get_handle (); +} + +void nano::store::read_transaction::reset () const +{ + impl->reset (); +} + +void nano::store::read_transaction::renew () const +{ + impl->renew (); +} + +void nano::store::read_transaction::refresh () const +{ + reset (); + renew (); +} + +nano::store::write_transaction::write_transaction (std::unique_ptr write_transaction_impl) : + impl (std::move (write_transaction_impl)) +{ + /* + * For IO threads, we do not want them to block on creating write transactions. + */ + debug_assert (nano::thread_role::get () != nano::thread_role::name::io); +} + +void * nano::store::write_transaction::get_handle () const +{ + return impl->get_handle (); +} + +void nano::store::write_transaction::commit () +{ + impl->commit (); +} + +void nano::store::write_transaction::renew () +{ + impl->renew (); +} + +void nano::store::write_transaction::refresh () +{ + impl->commit (); + impl->renew (); +} + +bool nano::store::write_transaction::contains (nano::tables table_a) const +{ + return impl->contains (table_a); +} diff --git a/nano/store/transaction.hpp b/nano/store/transaction.hpp new file mode 100644 index 0000000000..5061886907 --- /dev/null +++ b/nano/store/transaction.hpp @@ -0,0 +1,72 @@ +#pragma once + +#include + +#include + +namespace nano::store +{ +class transaction_impl +{ +public: + virtual ~transaction_impl () = default; + virtual void * get_handle () const = 0; +}; + +class read_transaction_impl : public transaction_impl +{ +public: + virtual void reset () = 0; + virtual void renew () = 0; +}; + +class write_transaction_impl : public transaction_impl +{ +public: + virtual void commit () = 0; + virtual void renew () = 0; + virtual bool contains (nano::tables table_a) const = 0; +}; + +class transaction +{ +public: + virtual ~transaction () = default; + virtual void * get_handle () const = 0; +}; + +/** + * RAII wrapper of a read MDB_txn where the constructor starts the transaction + * and the destructor aborts it. + */ +class read_transaction final : public transaction +{ +public: + explicit read_transaction (std::unique_ptr read_transaction_impl); + void * get_handle () const override; + void reset () const; + void renew () const; + void refresh () const; + +private: + std::unique_ptr impl; +}; + +/** + * RAII wrapper of a read-write MDB_txn where the constructor starts the transaction + * and the destructor commits it. + */ +class write_transaction final : public transaction +{ +public: + explicit write_transaction (std::unique_ptr write_transaction_impl); + void * get_handle () const override; + void commit (); + void renew (); + void refresh (); + bool contains (nano::tables table_a) const; + +private: + std::unique_ptr impl; +}; +} // namespace nano::store diff --git a/nano/store/version.cpp b/nano/store/version.cpp new file mode 100644 index 0000000000..d8917be835 --- /dev/null +++ b/nano/store/version.cpp @@ -0,0 +1 @@ +#include diff --git a/nano/store/version.hpp b/nano/store/version.hpp new file mode 100644 index 0000000000..79b294e3b0 --- /dev/null +++ b/nano/store/version.hpp @@ -0,0 +1,23 @@ +#pragma once + +#include +#include + +#include + +namespace nano +{ +class block_hash; +} +namespace nano::store +{ +/** + * Manages version storage + */ +class version +{ +public: + virtual void put (store::write_transaction const &, int) = 0; + virtual int get (store::transaction const &) const = 0; +}; +} // namespace nano::store diff --git a/nano/store/versioning.cpp b/nano/store/versioning.cpp new file mode 100644 index 0000000000..16433e0982 --- /dev/null +++ b/nano/store/versioning.cpp @@ -0,0 +1,5 @@ +#include + +#include + +#include diff --git a/nano/store/versioning.hpp b/nano/store/versioning.hpp new file mode 100644 index 0000000000..0160535345 --- /dev/null +++ b/nano/store/versioning.hpp @@ -0,0 +1,11 @@ +#pragma once + +#include +#include + +struct MDB_val; + +namespace nano::store +{ +// Holds historical verisons of classes used when upgrading database stores. +} // namespace nano::store diff --git a/nano/test_common/ledger.cpp b/nano/test_common/ledger.cpp index c87c0f8fe8..a3337bc925 100644 --- a/nano/test_common/ledger.cpp +++ b/nano/test_common/ledger.cpp @@ -1,3 +1,4 @@ +#include #include #include @@ -21,7 +22,7 @@ nano::ledger & nano::test::context::ledger_context::ledger () return ledger_m; } -nano::store & nano::test::context::ledger_context::store () +nano::store::component & nano::test::context::ledger_context::store () { return *store_m; } diff --git a/nano/test_common/ledger.hpp b/nano/test_common/ledger.hpp index 59ce59d78b..17bafae687 100644 --- a/nano/test_common/ledger.hpp +++ b/nano/test_common/ledger.hpp @@ -7,7 +7,10 @@ namespace nano { -class store; +namespace store +{ + class component; +} namespace test { namespace context @@ -19,13 +22,13 @@ namespace test Blocks must all return process_result::progress when processed */ ledger_context (std::deque> && blocks = std::deque>{}); nano::ledger & ledger (); - nano::store & store (); + nano::store::component & store (); nano::stats & stats (); std::deque> const & blocks () const; private: nano::logger_mt logger; - std::unique_ptr store_m; + std::unique_ptr store_m; nano::stats stats_m; nano::ledger ledger_m; std::deque> blocks_m; diff --git a/nano/test_common/system.cpp b/nano/test_common/system.cpp index 4ab1044402..044813fd69 100644 --- a/nano/test_common/system.cpp +++ b/nano/test_common/system.cpp @@ -203,7 +203,7 @@ std::shared_ptr nano::test::system::wallet (size_t index_a) return nodes[index_a]->wallets.items.begin ()->second; } -nano::account nano::test::system::account (nano::transaction const & transaction_a, size_t index_a) +nano::account nano::test::system::account (store::transaction const & transaction_a, size_t index_a) { auto wallet_l (wallet (index_a)); auto keys (wallet_l->store.begin (transaction_a)); @@ -451,7 +451,7 @@ nano::account nano::test::system::get_random_account (std::vector return result; } -nano::uint128_t nano::test::system::get_random_amount (nano::transaction const & transaction_a, nano::node & node_a, nano::account const & account_a) +nano::uint128_t nano::test::system::get_random_amount (store::transaction const & transaction_a, nano::node & node_a, nano::account const & account_a) { nano::uint128_t balance (node_a.ledger.account_balance (transaction_a, account_a)); nano::uint128_union random_amount; @@ -468,7 +468,7 @@ void nano::test::system::generate_send_existing (nano::node & node_a, std::vecto nano::account account; random_pool::generate_block (account.bytes.data (), sizeof (account.bytes)); auto transaction (node_a.store.tx_begin_read ()); - nano::store_iterator entry (node_a.store.account.begin (transaction, account)); + store::iterator entry (node_a.store.account.begin (transaction, account)); if (entry == node_a.store.account.end ()) { entry = node_a.store.account.begin (transaction); diff --git a/nano/test_common/system.hpp b/nano/test_common/system.hpp index b85f07fc8c..e03532ccef 100644 --- a/nano/test_common/system.hpp +++ b/nano/test_common/system.hpp @@ -30,7 +30,7 @@ namespace test void generate_usage_traffic (uint32_t, uint32_t, size_t); void generate_usage_traffic (uint32_t, uint32_t); nano::account get_random_account (std::vector &); - nano::uint128_t get_random_amount (nano::transaction const &, nano::node &, nano::account const &); + nano::uint128_t get_random_amount (store::transaction const &, nano::node &, nano::account const &); void generate_rollback (nano::node &, std::vector &); void generate_change_known (nano::node &, std::vector &); void generate_change_unknown (nano::node &, std::vector &); @@ -39,7 +39,7 @@ namespace test void generate_send_existing (nano::node &, std::vector &); std::unique_ptr upgrade_genesis_epoch (nano::node &, nano::epoch const); std::shared_ptr wallet (size_t); - nano::account account (nano::transaction const &, size_t); + nano::account account (store::transaction const &, size_t); /** Generate work with difficulty between \p min_difficulty_a (inclusive) and \p max_difficulty_a (exclusive) */ uint64_t work_generate_limited (nano::block_hash const & root_a, uint64_t min_difficulty_a, uint64_t max_difficulty_a); /** diff --git a/submodules/lmdb b/submodules/lmdb index e931980823..c7b3cc4df6 160000 --- a/submodules/lmdb +++ b/submodules/lmdb @@ -1 +1 @@ -Subproject commit e93198082347098e4222c28309c399bc17ac11a9 +Subproject commit c7b3cc4df6dfe8f0772fb509bdc74777667caa43