Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Block split - WIP #4295

Draft
wants to merge 7 commits into
base: develop
Choose a base branch
from
Draft

Block split - WIP #4295

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
d2d01fe
Moving block successors to their own table.
clemahieu Sep 25, 2023
56e1ef7
Block table split WIP
clemahieu Sep 25, 2023
d2feef0
Addressing missing return error.
clemahieu Sep 25, 2023
cee79e8
Fixing QT build.
clemahieu Sep 25, 2023
44c60d8
Disabling random block test.
clemahieu Sep 26, 2023
5899688
Using MDB_APPEND for block data insertion.
clemahieu Sep 26, 2023
1d34d85
Restore the highest block index from disk when node loads.
clemahieu Oct 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 9 additions & 44 deletions nano/core_test/block_store.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,6 @@ TEST (block_store, sideband_serialization)
sideband1.account = 1;
sideband1.balance = 2;
sideband1.height = 3;
sideband1.successor = 4;
sideband1.timestamp = 5;
std::vector<uint8_t> vector;
{
Expand All @@ -98,7 +97,6 @@ TEST (block_store, sideband_serialization)
ASSERT_EQ (sideband1.account, sideband2.account);
ASSERT_EQ (sideband1.balance, sideband2.balance);
ASSERT_EQ (sideband1.height, sideband2.height);
ASSERT_EQ (sideband1.successor, sideband2.successor);
ASSERT_EQ (sideband1.timestamp, sideband2.timestamp);
}

Expand Down Expand Up @@ -137,47 +135,14 @@ TEST (block_store, clear_successor)
{
nano::logger_mt logger;
auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants);
ASSERT_TRUE (!store->init_error ());
nano::block_builder builder;
auto block1 = builder
.open ()
.source (0)
.representative (1)
.account (0)
.sign (nano::keypair ().prv, 0)
.work (0)
.build ();
block1->sideband_set ({});
auto transaction (store->tx_begin_write ());
store->block.put (transaction, block1->hash (), *block1);
auto block2 = builder
.open ()
.source (0)
.representative (2)
.account (0)
.sign (nano::keypair ().prv, 0)
.work (0)
.build ();
block2->sideband_set ({});
store->block.put (transaction, block2->hash (), *block2);
auto block2_store (store->block.get (transaction, block1->hash ()));
ASSERT_NE (nullptr, block2_store);
ASSERT_EQ (0, block2_store->sideband ().successor.number ());
auto modified_sideband = block2_store->sideband ();
modified_sideband.successor = block2->hash ();
block1->sideband_set (modified_sideband);
store->block.put (transaction, block1->hash (), *block1);
{
auto block1_store (store->block.get (transaction, block1->hash ()));
ASSERT_NE (nullptr, block1_store);
ASSERT_EQ (block2->hash (), block1_store->sideband ().successor);
}
store->block.successor_clear (transaction, block1->hash ());
{
auto block1_store (store->block.get (transaction, block1->hash ()));
ASSERT_NE (nullptr, block1_store);
ASSERT_EQ (0, block1_store->sideband ().successor.number ());
}
nano::block_hash one{ 1 };
nano::block_hash two{ 2 };
auto tx = store->tx_begin_write ();
ASSERT_EQ (0, store->successor.get (tx, one));
store->successor.put (tx, one, two);
ASSERT_EQ (two, store->successor.get (tx, one));
store->successor.del (tx, one);
ASSERT_EQ (0, store->successor.get (tx, one));
}

TEST (block_store, add_nonempty_block)
Expand Down Expand Up @@ -898,7 +863,7 @@ TEST (block_store, cemented_count_cache)
ASSERT_EQ (1, ledger_cache.cemented_count);
}

TEST (block_store, block_random)
TEST (block_store, DISABLED_block_random)
{
nano::logger_mt logger;
auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants);
Expand Down
8 changes: 4 additions & 4 deletions nano/core_test/ledger.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -266,7 +266,7 @@ TEST (ledger, process_receive)
ASSERT_EQ (nano::dev::constants.genesis_amount - 25, ledger.account_balance (transaction, key2.pub));
ASSERT_EQ (nano::dev::constants.genesis_amount - 25, ledger.weight (key3.pub));
ASSERT_FALSE (ledger.rollback (transaction, hash4));
ASSERT_TRUE (store.block.successor (transaction, hash2).is_zero ());
ASSERT_TRUE (store.successor.get (transaction, hash2).is_zero ());
ASSERT_EQ (key2.pub, store.frontier.get (transaction, hash2));
ASSERT_TRUE (store.frontier.get (transaction, hash4).is_zero ());
ASSERT_EQ (25, ledger.account_balance (transaction, nano::dev::genesis_key.pub));
Expand Down Expand Up @@ -3109,7 +3109,7 @@ TEST (ledger, state_rollback_send)
ASSERT_EQ (nano::dev::constants.genesis_amount, ledger.account_balance (transaction, nano::dev::genesis->account ()));
ASSERT_EQ (nano::dev::constants.genesis_amount, ledger.weight (nano::dev::genesis->account ()));
ASSERT_FALSE (store.pending.exists (transaction, nano::pending_key (nano::dev::genesis->account (), send1->hash ())));
ASSERT_TRUE (store.block.successor (transaction, nano::dev::genesis->hash ()).is_zero ());
ASSERT_TRUE (store.successor.get (transaction, nano::dev::genesis->hash ()).is_zero ());
ASSERT_EQ (store.account.count (transaction), ledger.cache.account_count);
}

Expand Down Expand Up @@ -5474,7 +5474,7 @@ TEST (ledger, pruning_safe_functions)
ASSERT_FALSE (error);
}

TEST (ledger, hash_root_random)
TEST (ledger, DISABLED_hash_root_random)
{
nano::logger_mt logger;
auto store = nano::make_store (logger, nano::unique_path (), nano::dev::constants);
Expand Down Expand Up @@ -5536,7 +5536,7 @@ TEST (ledger, hash_root_random)
}
}

TEST (ledger, migrate_lmdb_to_rocksdb)
TEST (ledger, DISABLED_migrate_lmdb_to_rocksdb)
{
nano::test::system system{};
auto path = nano::unique_path ();
Expand Down
9 changes: 2 additions & 7 deletions nano/lib/blocks.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1754,8 +1754,7 @@ std::string nano::state_subtype (nano::block_details const details_a)
}
}

nano::block_sideband::block_sideband (nano::account const & account_a, nano::block_hash const & successor_a, nano::amount const & balance_a, uint64_t const height_a, nano::seconds_t const timestamp_a, nano::block_details const & details_a, nano::epoch const source_epoch_a) :
successor (successor_a),
nano::block_sideband::block_sideband (nano::account const & account_a, nano::amount const & balance_a, uint64_t const height_a, nano::seconds_t const timestamp_a, nano::block_details const & details_a, nano::epoch const source_epoch_a) :
account (account_a),
balance (balance_a),
height (height_a),
Expand All @@ -1765,8 +1764,7 @@ nano::block_sideband::block_sideband (nano::account const & account_a, nano::blo
{
}

nano::block_sideband::block_sideband (nano::account const & account_a, nano::block_hash const & successor_a, nano::amount const & balance_a, uint64_t const height_a, nano::seconds_t const timestamp_a, nano::epoch const epoch_a, bool const is_send, bool const is_receive, bool const is_epoch, nano::epoch const source_epoch_a) :
successor (successor_a),
nano::block_sideband::block_sideband (nano::account const & account_a, nano::amount const & balance_a, uint64_t const height_a, nano::seconds_t const timestamp_a, nano::epoch const epoch_a, bool const is_send, bool const is_receive, bool const is_epoch, nano::epoch const source_epoch_a) :
account (account_a),
balance (balance_a),
height (height_a),
Expand All @@ -1779,7 +1777,6 @@ nano::block_sideband::block_sideband (nano::account const & account_a, nano::blo
size_t nano::block_sideband::size (nano::block_type type_a)
{
size_t result (0);
result += sizeof (successor);
if (type_a != nano::block_type::state && type_a != nano::block_type::open)
{
result += sizeof (account);
Expand All @@ -1803,7 +1800,6 @@ size_t nano::block_sideband::size (nano::block_type type_a)

void nano::block_sideband::serialize (nano::stream & stream_a, nano::block_type type_a) const
{
nano::write (stream_a, successor.bytes);
if (type_a != nano::block_type::state && type_a != nano::block_type::open)
{
nano::write (stream_a, account.bytes);
Expand All @@ -1829,7 +1825,6 @@ bool nano::block_sideband::deserialize (nano::stream & stream_a, nano::block_typ
bool result (false);
try
{
nano::read (stream_a, successor.bytes);
if (type_a != nano::block_type::state && type_a != nano::block_type::open)
{
nano::read (stream_a, account.bytes);
Expand Down
5 changes: 2 additions & 3 deletions nano/lib/blocks.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,12 +59,11 @@ class block_sideband final
{
public:
block_sideband () = default;
block_sideband (nano::account const &, nano::block_hash const &, nano::amount const &, uint64_t const, nano::seconds_t const local_timestamp, nano::block_details const &, nano::epoch const source_epoch_a);
block_sideband (nano::account const &, nano::block_hash const &, nano::amount const &, uint64_t const, nano::seconds_t const local_timestamp, nano::epoch const epoch_a, bool const is_send, bool const is_receive, bool const is_epoch, nano::epoch const source_epoch_a);
block_sideband (nano::account const &, nano::amount const &, uint64_t const, nano::seconds_t const local_timestamp, nano::block_details const &, nano::epoch const source_epoch_a);
block_sideband (nano::account const &, nano::amount const &, uint64_t const, nano::seconds_t const local_timestamp, nano::epoch const epoch_a, bool const is_send, bool const is_receive, bool const is_epoch, nano::epoch const source_epoch_a);
void serialize (nano::stream &, nano::block_type) const;
bool deserialize (nano::stream &, nano::block_type);
static size_t size (nano::block_type);
nano::block_hash successor{ 0 };
nano::account account{};
nano::amount balance{ 0 };
uint64_t height{ 0 };
Expand Down
2 changes: 1 addition & 1 deletion nano/nano_node/entry.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1600,7 +1600,7 @@ int main (int argc, char * const * argv)
calculated_representative = block->representative ();
}
// Retrieving successor block hash
hash = node->store.block.successor (transaction, hash);
hash = node->store.successor.get (transaction, hash);
// Retrieving block data
if (!hash.is_zero ())
{
Expand Down
4 changes: 2 additions & 2 deletions nano/node/bootstrap/bootstrap_bulk_pull.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -393,7 +393,7 @@ void nano::bulk_pull_server::set_current_end ()
node->logger.try_log (boost::str (boost::format ("Bulk pull request for block hash: %1%") % request->start.to_string ()));
}

current = ascending () ? node->store.block.successor (transaction, request->start.as_block_hash ()) : request->start.as_block_hash ();
current = ascending () ? node->store.successor.get (transaction, request->start.as_block_hash ()) : request->start.as_block_hash ();
include_start = true;
}
else
Expand Down Expand Up @@ -514,7 +514,7 @@ std::shared_ptr<nano::block> nano::bulk_pull_server::get_next ()
result = node->block (current);
if (result != nullptr && set_current_to_end == false)
{
auto next = ascending () ? result->sideband ().successor : result->previous ();
auto next = ascending () ? node->successor (current) : result->previous ();
if (!next.is_zero ())
{
current = next;
Expand Down
5 changes: 2 additions & 3 deletions nano/node/bootstrap/bootstrap_server.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
#include <nano/store/block.hpp>
#include <nano/store/component.hpp>
#include <nano/store/confirmation_height.hpp>
#include <nano/store/successor.hpp>

// TODO: Make threads configurable
nano::bootstrap_server::bootstrap_server (nano::store::component & store_a, nano::ledger & ledger_a, nano::network_constants const & network_constants_a, nano::stats & stats_a) :
Expand Down Expand Up @@ -244,9 +245,7 @@ std::vector<std::shared_ptr<nano::block>> nano::bootstrap_server::prepare_blocks
while (current && result.size () < count)
{
result.push_back (current);

auto successor = current->sideband ().successor;
current = store.block.get (transaction, successor);
current = store.block.get (transaction, store.successor.get (transaction, current->hash ()));
}
}
return result;
Expand Down
15 changes: 8 additions & 7 deletions nano/node/confirmation_height_bounded.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
#include <nano/store/block.hpp>
#include <nano/store/confirmation_height.hpp>
#include <nano/store/pruned.hpp>
#include <nano/store/successor.hpp>

#include <boost/format.hpp>

Expand Down Expand Up @@ -233,7 +234,7 @@ nano::block_hash nano::confirmation_height_bounded::get_least_unconfirmed_hash_f
{
auto block (ledger.store.block.get (transaction_a, confirmation_height_info_a.frontier));
release_assert (block != nullptr);
least_unconfirmed_hash = block->sideband ().successor;
least_unconfirmed_hash = ledger.store.successor.get (transaction_a, confirmation_height_info_a.frontier);
block_height_a = block->sideband ().height + 1;
}
}
Expand All @@ -260,6 +261,7 @@ bool nano::confirmation_height_bounded::iterate (store::read_transaction const &
// Once a receive is cemented, we can cement all blocks above it until the next receive, so store those details for later.
++num_blocks;
auto block = ledger.store.block.get (transaction_a, hash);
auto successor = ledger.store.successor.get (transaction_a, hash);
auto source (block->source ());
if (source.is_zero ())
{
Expand All @@ -270,9 +272,8 @@ bool nano::confirmation_height_bounded::iterate (store::read_transaction const &
{
hit_receive = true;
reached_target = true;
auto const & sideband (block->sideband ());
auto next = !sideband.successor.is_zero () && sideband.successor != top_level_hash_a ? boost::optional<nano::block_hash> (sideband.successor) : boost::none;
receive_source_pairs_a.push_back ({ receive_chain_details{ account_a, sideband.height, hash, top_level_hash_a, next, bottom_height_a, bottom_hash_a }, source });
auto next = !successor.is_zero () && successor != top_level_hash_a ? boost::optional<nano::block_hash> (successor) : boost::none;
receive_source_pairs_a.push_back ({ receive_chain_details{ account_a, block->sideband ().height, hash, top_level_hash_a, next, bottom_height_a, bottom_hash_a }, source });
// Store a checkpoint every max_items so that we can always traverse a long number of accounts to genesis
if (receive_source_pairs_a.size () % max_items == 0)
{
Expand All @@ -289,7 +290,7 @@ bool nano::confirmation_height_bounded::iterate (store::read_transaction const &
}
else
{
hash = block->sideband ().successor;
hash = successor;
}
}

Expand Down Expand Up @@ -419,7 +420,7 @@ void nano::confirmation_height_bounded::cement_blocks (nano::write_guard & scope
else
{
auto block = ledger.store.block.get (transaction, confirmation_height_info.frontier);
new_cemented_frontier = block->sideband ().successor;
new_cemented_frontier = ledger.store.successor.get (transaction, confirmation_height_info.frontier);
num_blocks_confirmed = pending.top_height - confirmation_height_info.height;
start_height = confirmation_height_info.height + 1;
}
Expand Down Expand Up @@ -488,7 +489,7 @@ void nano::confirmation_height_bounded::cement_blocks (nano::write_guard & scope
// Get the next block in the chain until we have reached the final desired one
if (!last_iteration)
{
new_cemented_frontier = block->sideband ().successor;
new_cemented_frontier = ledger.store.successor.get (transaction, new_cemented_frontier);
block = ledger.store.block.get (transaction, new_cemented_frontier);
}
else
Expand Down
10 changes: 5 additions & 5 deletions nano/node/json_handler.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1160,7 +1160,7 @@ void nano::json_handler::block_info ()
response_l.put ("balance", balance.convert_to<std::string> ());
response_l.put ("height", std::to_string (block->sideband ().height));
response_l.put ("local_timestamp", std::to_string (block->sideband ().timestamp));
response_l.put ("successor", block->sideband ().successor.to_string ());
response_l.put ("successor", node.store.successor.get (transaction, hash).to_string ());
auto confirmed (node.ledger.block_confirmed (transaction, hash));
response_l.put ("confirmed", confirmed);

Expand Down Expand Up @@ -1320,7 +1320,7 @@ void nano::json_handler::blocks_info ()
entry.put ("balance", balance.convert_to<std::string> ());
entry.put ("height", std::to_string (block->sideband ().height));
entry.put ("local_timestamp", std::to_string (block->sideband ().timestamp));
entry.put ("successor", block->sideband ().successor.to_string ());
entry.put ("successor", node.store.successor.get (transaction, hash).to_string ());
auto confirmed (node.ledger.block_confirmed (transaction, hash));
entry.put ("confirmed", confirmed);

Expand Down Expand Up @@ -1959,7 +1959,7 @@ void nano::json_handler::chain (bool successors)
entry.put ("", hash.to_string ());
blocks.push_back (std::make_pair ("", entry));
}
hash = successors ? node.store.block.successor (transaction, hash) : block_l->previous ();
hash = successors ? node.store.successor.get (transaction, hash) : block_l->previous ();
}
else
{
Expand Down Expand Up @@ -2671,7 +2671,7 @@ void nano::json_handler::account_history ()
--count;
}
}
hash = reverse ? node.store.block.successor (transaction, hash) : block->previous ();
hash = reverse ? node.store.successor.get (transaction, hash) : block->previous ();
block = node.store.block.get (transaction, hash);
}
response_l.add_child ("history", history);
Expand Down Expand Up @@ -3706,7 +3706,7 @@ void nano::json_handler::republish ()
}
}
}
hash = node.store.block.successor (transaction, hash);
hash = node.store.successor.get (transaction, hash);
}
node.network.flood_block_many (std::move (republish_bundle), nullptr, 25);
response_l.put ("success", ""); // obsolete
Expand Down
6 changes: 6 additions & 0 deletions nano/node/node.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -773,6 +773,12 @@ std::shared_ptr<nano::block> nano::node::block (nano::block_hash const & hash_a)
return store.block.get (transaction, hash_a);
}

nano::block_hash nano::node::successor (nano::block_hash const & hash)
{
auto tx = store.tx_begin_read ();
return store.successor.get (tx, hash);
}

std::pair<nano::uint128_t, nano::uint128_t> nano::node::balance_pending (nano::account const & account_a, bool only_confirmed_a)
{
std::pair<nano::uint128_t, nano::uint128_t> result;
Expand Down
1 change: 1 addition & 0 deletions nano/node/node.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ class node final : public std::enable_shared_from_this<nano::node>
void process_local_async (std::shared_ptr<nano::block> const &);
void keepalive_preconfigured (std::vector<std::string> const &);
std::shared_ptr<nano::block> block (nano::block_hash const &);
nano::block_hash successor (nano::block_hash const & hash);
std::pair<nano::uint128_t, nano::uint128_t> balance_pending (nano::account const &, bool only_confirmed);
nano::uint128_t weight (nano::account const &);
nano::block_hash rep_block (nano::account const &);
Expand Down
2 changes: 1 addition & 1 deletion nano/node/request_aggregator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -237,7 +237,7 @@ std::pair<std::vector<std::shared_ptr<nano::block>>, std::vector<std::shared_ptr
if (block == nullptr && !root.is_zero ())
{
// Search for block root
auto successor (ledger.store.block.successor (transaction, root.as_block_hash ()));
auto successor (ledger.store.successor.get (transaction, root.as_block_hash ()));

// Search for account root
if (successor.is_zero ())
Expand Down
2 changes: 1 addition & 1 deletion nano/node/scheduler/priority.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ bool nano::scheduler::priority::activate (nano::account const & account_a, store
if (conf_info.height < info->block_count)
{
debug_assert (conf_info.frontier != info->head);
auto hash = conf_info.height == 0 ? info->open_block : node.store.block.successor (transaction, conf_info.frontier);
auto hash = conf_info.height == 0 ? info->open_block : node.store.successor.get (transaction, conf_info.frontier);
auto block = node.store.block.get (transaction, hash);
debug_assert (block != nullptr);
if (node.ledger.dependents_confirmed (transaction, *block))
Expand Down
4 changes: 2 additions & 2 deletions nano/qt/qt.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -677,7 +677,7 @@ nano_qt::block_viewer::block_viewer (nano_qt::wallet & wallet_a) :
std::string contents;
block_l->serialize_json (contents);
block->setPlainText (contents.c_str ());
auto successor_l (this->wallet.node.store.block.successor (transaction, hash_l));
auto successor_l (this->wallet.node.store.successor.get (transaction, hash_l));
successor->setText (successor_l.to_string ().c_str ());
}
else
Expand Down Expand Up @@ -721,7 +721,7 @@ void nano_qt::block_viewer::rebroadcast_action (nano::block_hash const & hash_a)
if (block != nullptr)
{
wallet.node.network.flood_block (block);
auto successor (wallet.node.store.block.successor (transaction, hash_a));
auto successor (wallet.node.store.successor.get (transaction, hash_a));
if (!successor.is_zero ())
{
done = false;
Expand Down
Loading
Loading