diff --git a/.gitmodules b/.gitmodules index d37697838b..6f9e8c84da 100644 --- a/.gitmodules +++ b/.gitmodules @@ -22,9 +22,6 @@ [submodule "rocksdb"] path = submodules/rocksdb url = https://github.com/facebook/rocksdb.git -[submodule "diskhash"] - path = submodules/diskhash - url = https://github.com/nanocurrency/diskhash.git [submodule "boost"] path = submodules/boost url = https://github.com/boostorg/boost.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 0c35f83697..f9a36b4b34 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -466,13 +466,6 @@ target_link_libraries(boost_property_tree INTERFACE Boost::any) target_link_libraries(boost_property_tree INTERFACE Boost::format) target_link_libraries(boost_property_tree INTERFACE Boost::multi_index) -# diskhash -if(NOT CMAKE_SYSTEM_NAME STREQUAL "Windows") - add_library(diskhash STATIC - ${CMAKE_SOURCE_DIR}/submodules/diskhash/src/diskhash.c) - include_directories(submodules/diskhash/src) -endif() - # RocksDB include_directories(submodules/rocksdb/include) set(USE_RTTI diff --git a/coverage/CMakeLists.txt b/coverage/CMakeLists.txt index 4039baa0bb..5ec5055371 100644 --- a/coverage/CMakeLists.txt +++ b/coverage/CMakeLists.txt @@ -31,7 +31,7 @@ add_custom_target( '${PROJECT_SOURCE_DIR}/submodules/flatbuffers/*' '${PROJECT_SOURCE_DIR}/gtest/*' '${PROJECT_SOURCE_DIR}/submodules/rocksdb/*' '${PROJECT_SOURCE_DIR}/valgrind/*' '${PROJECT_SOURCE_DIR}/nano/core_test/*' - '${PROJECT_SOURCE_DIR}/diskhash/*' '${PROJECT_SOURCE_DIR}/nano/load_test/*' + '${PROJECT_SOURCE_DIR}/nano/load_test/*' '${PROJECT_SOURCE_DIR}/nano/ipc_flatbuffers_test/*' '${PROJECT_SOURCE_DIR}/nano/ipc_flatbuffers_lib/*' '${PROJECT_SOURCE_DIR}/nano/nano_node/*' diff --git a/nano/core_test/CMakeLists.txt b/nano/core_test/CMakeLists.txt index 0897fd7b11..059d15acd1 100644 --- a/nano/core_test/CMakeLists.txt +++ b/nano/core_test/CMakeLists.txt @@ -25,7 +25,6 @@ add_executable( gap_cache.cpp ipc.cpp ledger.cpp - ledger_walker.cpp locks.cpp logger.cpp message.cpp diff --git a/nano/core_test/ledger_walker.cpp b/nano/core_test/ledger_walker.cpp deleted file mode 100644 index 29797a212d..0000000000 --- a/nano/core_test/ledger_walker.cpp +++ /dev/null @@ -1,226 +0,0 @@ -#include -#include -#include - -#include - -#include - -// TODO: keep this until diskhash builds fine on Windows -#ifndef _WIN32 - -using namespace std::chrono_literals; - -TEST (ledger_walker, genesis_block) -{ - nano::test::system system{}; - auto const node = system.add_node (); - - nano::ledger_walker ledger_walker{ node->ledger }; - - std::size_t walked_blocks_count = 0; - ledger_walker.walk_backward (nano::dev::genesis->hash (), - [&] (auto const & block) { - ++walked_blocks_count; - EXPECT_EQ (block->hash (), nano::dev::genesis->hash ()); - }); - - EXPECT_EQ (walked_blocks_count, 1); - - walked_blocks_count = 0; - ledger_walker.walk (nano::dev::genesis->hash (), - [&] (auto const & block) { - ++walked_blocks_count; - EXPECT_EQ (block->hash (), nano::dev::genesis->hash ()); - }); - - EXPECT_EQ (walked_blocks_count, 1); -} - -namespace nano -{ -TEST (ledger_walker, genesis_account_longer) -{ - nano::test::system system{}; - nano::node_config node_config = system.default_config (); - node_config.enable_voting = true; - node_config.receive_minimum = 1; - - auto const node = system.add_node (node_config); - - nano::ledger_walker ledger_walker{ node->ledger }; - EXPECT_TRUE (ledger_walker.walked_blocks.empty ()); - EXPECT_LE (ledger_walker.walked_blocks.bucket_count (), 1); - EXPECT_TRUE (ledger_walker.blocks_to_walk.empty ()); - - auto const get_number_of_walked_blocks = [&ledger_walker] (auto const & start_block_hash) { - std::size_t walked_blocks_count = 0; - ledger_walker.walk_backward (start_block_hash, - [&] (auto const & block) { - ++walked_blocks_count; - }); - - return walked_blocks_count; - }; - - auto const transaction = node->ledger.store.tx_begin_read (); - auto genesis_account_info = node->ledger.account_info (transaction, nano::dev::genesis_key.pub); - ASSERT_TRUE (genesis_account_info); - EXPECT_EQ (get_number_of_walked_blocks (genesis_account_info->open_block), 1); - EXPECT_EQ (get_number_of_walked_blocks (genesis_account_info->head), 1); - - system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); - for (auto itr = 1; itr <= 5; ++itr) - { - auto const send = system.wallet (0)->send_action (nano::dev::genesis_key.pub, nano::dev::genesis_key.pub, 1); - ASSERT_TRUE (send); - EXPECT_EQ (get_number_of_walked_blocks (send->hash ()), 1 + itr * 2 - 1); - ASSERT_TIMELY (3s, 1 + itr * 2 == node->ledger.cache.cemented_count); - ASSERT_TRUE (node->ledger.account_info (transaction, nano::dev::genesis_key.pub)); - // TODO: check issue with account head - // EXPECT_EQ(get_number_of_walked_blocks (genesis_account_info.head), 1 + itr * 2); - } - - EXPECT_TRUE (ledger_walker.walked_blocks.empty ()); - EXPECT_LE (ledger_walker.walked_blocks.bucket_count (), 1); - EXPECT_TRUE (ledger_walker.blocks_to_walk.empty ()); -} - -} - -TEST (ledger_walker, cross_account) -{ - nano::test::system system{}; - nano::node_config node_config = system.default_config (); - node_config.enable_voting = true; - node_config.receive_minimum = 1; - - auto const node = system.add_node (node_config); - - system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); - ASSERT_TRUE (system.wallet (0)->send_action (nano::dev::genesis_key.pub, nano::dev::genesis_key.pub, 1)); - ASSERT_TIMELY (3s, 3 == node->ledger.cache.cemented_count); - - nano::keypair key{}; - system.wallet (0)->insert_adhoc (key.prv); - ASSERT_TRUE (system.wallet (0)->send_action (nano::dev::genesis_key.pub, key.pub, 1)); - ASSERT_TIMELY (3s, 5 == node->ledger.cache.cemented_count); - - auto const transaction = node->ledger.store.tx_begin_read (); - auto account_info = node->ledger.account_info (transaction, key.pub); - ASSERT_TRUE (account_info); - - // TODO: check issue with account head - // auto const first = node->ledger.store.block.get(transaction, account_info.head); - // auto const second = node->ledger.store.block.get(transaction, first->previous()); - // auto const third = node->ledger.store.block.get(transaction, second->previous()); - // auto const fourth = node->ledger.store.block.get(transaction, third->previous()); - // auto const fifth = node->ledger.store.block.get(transaction, fourth->previous()); - // - // auto const expected_blocks_to_walk = { first, second, third, fourth, fifth }; - // auto expected_blocks_to_walk_itr = expected_blocks_to_walk.begin(); - // - // nano::ledger_walker ledger_walker{ node->ledger }; - // ledger_walker.walk_backward (account_info.block_count, [&] (auto const & block) { - // if (expected_blocks_to_walk_itr == expected_blocks_to_walk.end()) - // { - // EXPECT_TRUE(false); - // return false; - // } - // - // EXPECT_EQ((*expected_blocks_to_walk_itr++)->hash(), block->hash()); - // return true; - // }); - // - // EXPECT_EQ(expected_blocks_to_walk_itr, expected_blocks_to_walk.end()); -} - -// Test disabled because it's failing intermittently. -// PR in which it got disabled: https://github.com/nanocurrency/nano-node/pull/3602 -// Issue for investigating it: https://github.com/nanocurrency/nano-node/issues/3603 -TEST (ledger_walker, DISABLED_ladder_geometry) -{ - nano::test::system system{}; - - nano::node_config node_config = system.default_config (); - node_config.enable_voting = true; - node_config.receive_minimum = 1; - - auto const node = system.add_node (node_config); - std::array keys{}; - - system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv); - for (auto itr = 0; itr != keys.size (); ++itr) - { - system.wallet (0)->insert_adhoc (keys[itr].prv); - auto const block = system.wallet (0)->send_action (nano::dev::genesis_key.pub, keys[itr].pub, 1000); - ASSERT_TRUE (block); - ASSERT_TIMELY (3s, 1 + (itr + 1) * 2 == node->ledger.cache.cemented_count); - } - - std::vector amounts_to_send (10); - std::iota (amounts_to_send.begin (), amounts_to_send.end (), 1); - - nano::account const * last_destination{}; - for (auto itr = 0; itr != amounts_to_send.size (); ++itr) - { - auto const source_index = itr % keys.size (); - auto const destination_index = (source_index + 1) % keys.size (); - last_destination = &keys[destination_index].pub; - - auto const send = system.wallet (0)->send_action (keys[source_index].pub, keys[destination_index].pub, amounts_to_send[itr]); - ASSERT_TRUE (send); - ASSERT_TIMELY (3s, 1 + keys.size () * 2 + (itr + 1) * 2 == node->ledger.cache.cemented_count); - } - - ASSERT_TRUE (last_destination); - auto last_destination_info = node->ledger.account_info (node->ledger.store.tx_begin_read (), *last_destination); - ASSERT_TRUE (last_destination_info); - - // This is how we expect chains to look like (for 3 accounts and 10 amounts to be sent) - // k1: 1000 SEND 3 SEND 6 SEND 9 SEND - // k2: 1000 1 SEND 4 SEND 7 SEND 10 - // k3: 1000 2 SEND 5 SEND 8 SEND - - std::vector amounts_expected_backwards{ 10, 9, 8, 5, 4, 3, 1000, 1, 1000, 2, 1000, 6, 7 }; - auto amounts_expected_backwards_itr = amounts_expected_backwards.cbegin (); - - nano::ledger_walker ledger_walker{ node->ledger }; - ledger_walker.walk_backward (last_destination_info->head, - [&] (auto const & block) { - if (block->sideband ().details.is_receive) - { - nano::amount previous_balance{}; - if (!block->previous ().is_zero ()) - { - auto const previous_block = node->ledger.store.block.get (node->ledger.store.tx_begin_read (), block->previous ()); - previous_balance = previous_block->balance (); - } - - EXPECT_EQ (*amounts_expected_backwards_itr++, block->balance ().number () - previous_balance.number ()); - } - }); - - EXPECT_EQ (amounts_expected_backwards_itr, amounts_expected_backwards.cend ()); - - auto amounts_expected_itr = amounts_expected_backwards.crbegin (); - - ledger_walker.walk (last_destination_info->head, - [&] (auto const & block) { - if (block->sideband ().details.is_receive) - { - nano::amount previous_balance{}; - if (!block->previous ().is_zero ()) - { - auto const previous_block = node->ledger.store.block.get (node->ledger.store.tx_begin_read (), block->previous ()); - previous_balance = previous_block->balance (); - } - - EXPECT_EQ (*amounts_expected_itr++, block->balance ().number () - previous_balance.number ()); - } - }); - - EXPECT_EQ (amounts_expected_itr, amounts_expected_backwards.crend ()); -} - -#endif // _WIN32 -- TODO: keep this until diskhash builds fine on Windows diff --git a/nano/node/CMakeLists.txt b/nano/node/CMakeLists.txt index 3846c78973..b4de400f8f 100644 --- a/nano/node/CMakeLists.txt +++ b/nano/node/CMakeLists.txt @@ -110,8 +110,6 @@ add_library( ipc/ipc_server.cpp json_handler.hpp json_handler.cpp - ledger_walker.hpp - ledger_walker.cpp logging.hpp logging.cpp make_store.hpp @@ -196,10 +194,6 @@ add_library( messages.cpp xorshift.hpp) -if(NOT CMAKE_SYSTEM_NAME STREQUAL "Windows") - set(DISKHASH diskhash) -endif() - target_link_libraries( node nano_lib @@ -217,7 +211,6 @@ target_link_libraries( Boost::system Boost::thread rocksdb - ${DISKHASH} ${CMAKE_DL_LIBS} ${psapi_lib}) diff --git a/nano/node/ledger_walker.cpp b/nano/node/ledger_walker.cpp deleted file mode 100644 index 075cd1728e..0000000000 --- a/nano/node/ledger_walker.cpp +++ /dev/null @@ -1,181 +0,0 @@ -// TODO: keep this until diskhash builds fine on Windows -#ifndef _WIN32 - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -nano::ledger_walker::ledger_walker (nano::ledger const & ledger_a) : - ledger{ ledger_a }, - use_in_memory_walked_blocks{ true }, - walked_blocks{}, - walked_blocks_disk{}, - blocks_to_walk{} -{ - debug_assert (!ledger.store.init_error ()); -} - -void nano::ledger_walker::walk_backward (nano::block_hash const & start_block_hash_a, should_visit_callback const & should_visit_callback_a, visitor_callback const & visitor_callback_a) -{ - auto const transaction = ledger.store.tx_begin_read (); - - enqueue_block (start_block_hash_a); - while (!blocks_to_walk.empty ()) - { - auto const block = dequeue_block (transaction); - if (!should_visit_callback_a (block)) - { - continue; - } - - visitor_callback_a (block); - for (auto const & hash : ledger.dependent_blocks (transaction, *block)) - { - if (!hash.is_zero ()) - { - auto const dependent_block = ledger.store.block.get (transaction, hash); - if (dependent_block) - { - enqueue_block (dependent_block); - } - } - } - } - - clear_queue (); -} - -void nano::ledger_walker::walk (nano::block_hash const & end_block_hash_a, should_visit_callback const & should_visit_callback_a, visitor_callback const & visitor_callback_a) -{ - std::uint64_t last_walked_block_order_index = 0; - dht::DiskHash walked_blocks_order{ nano::unique_path ().c_str (), static_cast (std::to_string (std::numeric_limits::max ()).size ()) + 1, dht::DHOpenRW }; - - walk_backward (end_block_hash_a, - should_visit_callback_a, - [&] (auto const & block) { - walked_blocks_order.insert (std::to_string (++last_walked_block_order_index).c_str (), block->hash ()); - }); - - auto const transaction = ledger.store.tx_begin_read (); - for (auto walked_block_order_index = last_walked_block_order_index; walked_block_order_index != 0; --walked_block_order_index) - { - auto const * block_hash = walked_blocks_order.lookup (std::to_string (walked_block_order_index).c_str ()); - if (!block_hash) - { - debug_assert (false); - continue; - } - - auto const block = ledger.store.block.get (transaction, *block_hash); - if (!block) - { - debug_assert (false); - continue; - } - - visitor_callback_a (block); - } -} - -void nano::ledger_walker::walk_backward (nano::block_hash const & start_block_hash_a, visitor_callback const & visitor_callback_a) -{ - walk_backward ( - start_block_hash_a, - [&] (auto const & /* block */) { - return true; - }, - visitor_callback_a); -} - -void nano::ledger_walker::walk (nano::block_hash const & end_block_hash_a, visitor_callback const & visitor_callback_a) -{ - walk ( - end_block_hash_a, - [&] (auto const & /* block */) { - return true; - }, - visitor_callback_a); -} - -void nano::ledger_walker::enqueue_block (nano::block_hash block_hash_a) -{ - if (add_to_walked_blocks (block_hash_a)) - { - blocks_to_walk.emplace (std::move (block_hash_a)); - } -} - -void nano::ledger_walker::enqueue_block (std::shared_ptr const & block_a) -{ - debug_assert (block_a); - enqueue_block (block_a->hash ()); -} - -bool nano::ledger_walker::add_to_walked_blocks (nano::block_hash const & block_hash_a) -{ - if (use_in_memory_walked_blocks) - { - if (walked_blocks.size () < in_memory_block_count) - { - return walked_blocks.emplace (block_hash_a).second; - } - - use_in_memory_walked_blocks = false; - - debug_assert (!walked_blocks_disk.has_value ()); - walked_blocks_disk.emplace (nano::unique_path ().c_str (), sizeof (nano::block_hash::bytes) + 1, dht::DHOpenRW); - - for (auto const & walked_block_hash : walked_blocks) - { - if (!add_to_walked_blocks_disk (walked_block_hash)) - { - debug_assert (false); - } - } - - decltype (walked_blocks){}.swap (walked_blocks); - } - - return add_to_walked_blocks_disk (block_hash_a); -} - -bool nano::ledger_walker::add_to_walked_blocks_disk (nano::block_hash const & block_hash_a) -{ - debug_assert (!use_in_memory_walked_blocks); - debug_assert (walked_blocks_disk.has_value ()); - - std::array block_hash_key{}; - std::copy (block_hash_a.chars.cbegin (), - block_hash_a.chars.cend (), - block_hash_key.begin ()); - - return walked_blocks_disk->insert (block_hash_key.data (), true); -} - -void nano::ledger_walker::clear_queue () -{ - use_in_memory_walked_blocks = true; - - decltype (walked_blocks){}.swap (walked_blocks); - walked_blocks_disk.reset (); - - decltype (blocks_to_walk){}.swap (blocks_to_walk); -} - -std::shared_ptr nano::ledger_walker::dequeue_block (store::transaction const & transaction_a) -{ - auto block = ledger.store.block.get (transaction_a, blocks_to_walk.top ()); - blocks_to_walk.pop (); - - return block; -} - -#endif // _WIN32 -- TODO: keep this until diskhash builds fine on Windows diff --git a/nano/node/ledger_walker.hpp b/nano/node/ledger_walker.hpp deleted file mode 100644 index 5492767da7..0000000000 --- a/nano/node/ledger_walker.hpp +++ /dev/null @@ -1,69 +0,0 @@ -// TODO: keep this until diskhash builds fine on Windows -#ifndef _WIN32 - -#pragma once - -#include - -#include -#include -#include -#include -#include -#include - -#include - -namespace nano::store -{ -class transaction; -} - -namespace nano -{ -class block; -class ledger; - -/** Walks the ledger starting from a start block and applying a depth-first search algorithm */ -class ledger_walker final -{ -public: - using should_visit_callback = std::function const &)>; - using visitor_callback = std::function const &)>; - - explicit ledger_walker (nano::ledger const & ledger_a); - - /** Start traversing (in a backwards direction -- towards genesis) from \p start_block_hash_a until \p should_visit_callback_a returns false, calling \p visitor_callback_a at each block. Prefer 'walk' instead, if possible. */ - void walk_backward (nano::block_hash const & start_block_hash_a, should_visit_callback const & should_visit_callback_a, visitor_callback const & visitor_callback_a); - - /** Start traversing (in a forward direction -- towards end_block_hash_a) from first block (genesis onwards) where \p should_visit_a returns true until \p end_block_hash_a, calling \p visitor_callback at each block. Prefer this one, instead of 'walk_backwards', if possible. */ - void walk (nano::block_hash const & end_block_hash_a, should_visit_callback const & should_visit_callback_a, visitor_callback const & visitor_callback_a); - - /** Methods similar to walk_backward and walk, but that do not offer the possibility of providing a user-defined should_visit_callback function. */ - void walk_backward (nano::block_hash const & start_block_hash_a, visitor_callback const & visitor_callback_a); - void walk (nano::block_hash const & end_block_hash_a, visitor_callback const & visitor_callback_a); - - /** How many blocks will be held in the in-memory hash before using the disk hash for walking. */ - // TODO TSB: make this 65536 - static constexpr std::size_t in_memory_block_count = 0; - -private: - nano::ledger const & ledger; - bool use_in_memory_walked_blocks; - std::unordered_set walked_blocks; - std::optional> walked_blocks_disk; - std::stack blocks_to_walk; - - void enqueue_block (nano::block_hash block_hash_a); - void enqueue_block (std::shared_ptr const & block_a); - bool add_to_walked_blocks (nano::block_hash const & block_hash_a); - bool add_to_walked_blocks_disk (nano::block_hash const & block_hash_a); - void clear_queue (); - std::shared_ptr dequeue_block (store::transaction const & transaction_a); - - friend class ledger_walker_genesis_account_longer_Test; -}; - -} - -#endif // _WIN32 -- TODO: keep this until diskhash builds fine on Windows