Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

migrate_database_lmdb_to_rocksdb improvements #4647

Merged
merged 12 commits into from
Jul 10, 2024
Merged

migrate_database_lmdb_to_rocksdb improvements #4647

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
073d6c3
Rocksdb migration: disk space check and progress output
RickiNano Jun 13, 2024
dce3b10
Estimate the required free disk space
RickiNano Jun 13, 2024
1aa3e6b
More granular progress feedback
RickiNano Jun 13, 2024
87f864c
Use Nano Logger
RickiNano Jun 14, 2024
b256c75
Clang format fix
RickiNano Jun 23, 2024
6ad9d73
Fixed double count
RickiNano Jun 26, 2024
98ce010
Use atomics for multithreaded counters are accurate
clemahieu Jul 1, 2024
e4568b3
Use a single periodically refreshed transaction per thread.
clemahieu Jul 2, 2024
1c3f089
Output the number of entries for each table
RickiNano Jul 3, 2024
010fd0e
Use TransactionDB rather than OptimisticTransactionDB.
clemahieu Jul 5, 2024
317f185
Use default refresh interval.
clemahieu Jul 9, 2024
bcb5369
Fixed error and aligned log output format
RickiNano Jul 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions nano/lib/logging_enums.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ enum class type
opencl_work,
upnp,
rep_crawler,
ledger,
lmdb,
rocksdb,
txn_tracker,
Expand Down
7 changes: 1 addition & 6 deletions nano/node/cli.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -483,19 +483,14 @@ std::error_code nano::handle_node_options (boost::program_options::variables_map
auto error (false);
if (!node.node->init_error ())
{
std::cout << "Migrating LMDB database to RocksDB, might take a while..." << std::endl;
error = node.node->ledger.migrate_lmdb_to_rocksdb (data_path);
}
else
{
error = true;
}

if (!error)
{
std::cout << "Migration completed, after confirming it is correct the data.ldb file can be deleted if no longer required" << std::endl;
}
else
if (error)
{
std::cerr << "There was an error migrating" << std::endl;
}
Expand Down
112 changes: 96 additions & 16 deletions nano/secure/ledger.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1241,12 +1241,25 @@ uint64_t nano::ledger::pruning_action (secure::write_transaction & transaction_a
// A precondition is that the store is an LMDB store
bool nano::ledger::migrate_lmdb_to_rocksdb (std::filesystem::path const & data_path_a) const
{
nano::logger logger;
nano::logger::initialize (nano::log_config::daemon_default (), data_path_a);

logger.info (nano::log::type::ledger, "Migrating LMDB database to RocksDB. This will take a while...");

std::filesystem::space_info si = std::filesystem::space (data_path_a);
auto file_size = std::filesystem::file_size (data_path_a / "data.ldb");
const auto estimated_required_space = file_size * 0.65; // RocksDb database size is approximately 65% of the lmdb size

if (si.available < estimated_required_space)
{
logger.warn (nano::log::type::ledger, "You may not have enough available disk space. Estimated free space requirement is {} GB", estimated_required_space / 1024 / 1024 / 1024);
}

boost::system::error_code error_chmod;
nano::set_secure_perm_directory (data_path_a, error_chmod);
auto rockdb_data_path = data_path_a / "rocksdb";
std::filesystem::remove_all (rockdb_data_path);

nano::logger logger;
auto error (false);

// Open rocksdb database
Expand All @@ -1256,76 +1269,140 @@ bool nano::ledger::migrate_lmdb_to_rocksdb (std::filesystem::path const & data_p

if (!rocksdb_store->init_error ())
{
auto table_size = store.count (store.tx_begin_read (), tables::blocks);
logger.info (nano::log::type::ledger, "Step 1 of 7: Converting {} entries from blocks table", table_size);
std::atomic<std::size_t> count = 0;
store.block.for_each_par (
[&rocksdb_store] (store::read_transaction const & /*unused*/, auto i, auto n) {
[&] (store::read_transaction const & /*unused*/, auto i, auto n) {
auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::blocks }));
for (; i != n; ++i)
{
auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::blocks }));

rocksdb_transaction.refresh_if_needed ();
std::vector<uint8_t> vector;
{
nano::vectorstream stream (vector);
nano::serialize_block (stream, *i->second.block);
i->second.sideband.serialize (stream, i->second.block->type ());
}
rocksdb_store->block.raw_put (rocksdb_transaction, vector, i->first);

if (auto count_l = ++count; count_l % 5000000 == 0)
{
logger.info (nano::log::type::ledger, "{} blocks converted", count_l);
}
}
});
logger.info (nano::log::type::ledger, "Finished converting {} blocks", count.load ());

table_size = store.count (store.tx_begin_read (), tables::pending);
logger.info (nano::log::type::ledger, "Step 2 of 7: Converting {} entries from pending table", table_size);
count = 0;
store.pending.for_each_par (
[&rocksdb_store] (store::read_transaction const & /*unused*/, auto i, auto n) {
[&] (store::read_transaction const & /*unused*/, auto i, auto n) {
auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::pending }));
for (; i != n; ++i)
{
auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::pending }));
rocksdb_transaction.refresh_if_needed ();
rocksdb_store->pending.put (rocksdb_transaction, i->first, i->second);
if (auto count_l = ++count; count_l % 500000 == 0)
{
logger.info (nano::log::type::ledger, "{} entries converted", count_l);
}
}
});
logger.info (nano::log::type::ledger, "Finished converting {} entries", count.load ());

table_size = store.count (store.tx_begin_read (), tables::confirmation_height);
logger.info (nano::log::type::ledger, "Step 3 of 7: Converting {} entries from confirmation_height table", table_size);
count = 0;
store.confirmation_height.for_each_par (
[&rocksdb_store] (store::read_transaction const & /*unused*/, auto i, auto n) {
[&] (store::read_transaction const & /*unused*/, auto i, auto n) {
auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::confirmation_height }));
for (; i != n; ++i)
{
auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::confirmation_height }));
rocksdb_transaction.refresh_if_needed ();
rocksdb_store->confirmation_height.put (rocksdb_transaction, i->first, i->second);
if (auto count_l = ++count; count_l % 500000 == 0)
{
logger.info (nano::log::type::ledger, "{} entries converted", count_l);
}
}
});
logger.info (nano::log::type::ledger, "Finished converting {} entries", count.load ());

table_size = store.count (store.tx_begin_read (), tables::accounts);
logger.info (nano::log::type::ledger, "Step 4 of 7: Converting {} entries from accounts table", table_size);
count = 0;
store.account.for_each_par (
[&rocksdb_store] (store::read_transaction const & /*unused*/, auto i, auto n) {
[&] (store::read_transaction const & /*unused*/, auto i, auto n) {
auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::accounts }));
for (; i != n; ++i)
{
auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::accounts }));
rocksdb_transaction.refresh_if_needed ();
rocksdb_store->account.put (rocksdb_transaction, i->first, i->second);
if (auto count_l = ++count; count_l % 500000 == 0)
{
logger.info (nano::log::type::ledger, "{} entries converted", count_l);
}
}
});
logger.info (nano::log::type::ledger, "Finished converting {} entries", count.load ());

table_size = store.count (store.tx_begin_read (), tables::rep_weights);
logger.info (nano::log::type::ledger, "Step 5 of 7: Converting {} entries from rep_weights table", table_size);
count = 0;
store.rep_weight.for_each_par (
[&rocksdb_store] (store::read_transaction const & /*unused*/, auto i, auto n) {
[&] (store::read_transaction const & /*unused*/, auto i, auto n) {
auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::rep_weights }));
for (; i != n; ++i)
{
auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::rep_weights }));
rocksdb_transaction.refresh_if_needed ();
rocksdb_store->rep_weight.put (rocksdb_transaction, i->first, i->second.number ());
if (auto count_l = ++count; count_l % 500000 == 0)
{
logger.info (nano::log::type::ledger, "{} entries converted", count_l);
}
}
});
logger.info (nano::log::type::ledger, "Finished converting {} entries", count.load ());

table_size = store.count (store.tx_begin_read (), tables::pruned);
logger.info (nano::log::type::ledger, "Step 6 of 7: Converting {} entries from pruned table", table_size);
count = 0;
store.pruned.for_each_par (
[&rocksdb_store] (store::read_transaction const & /*unused*/, auto i, auto n) {
[&] (store::read_transaction const & /*unused*/, auto i, auto n) {
auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::pruned }));
for (; i != n; ++i)
{
auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::pruned }));
rocksdb_transaction.refresh_if_needed ();
rocksdb_store->pruned.put (rocksdb_transaction, i->first);
if (auto count_l = ++count; count_l % 500000 == 0)
{
logger.info (nano::log::type::ledger, "{} entries converted", count_l);
}
}
});
logger.info (nano::log::type::ledger, "Finished converting {} entries", count.load ());

table_size = store.count (store.tx_begin_read (), tables::final_votes);
logger.info (nano::log::type::ledger, "Step 7 of 7: Converting {} entries from final_votes table", table_size);
count = 0;
store.final_vote.for_each_par (
[&rocksdb_store] (store::read_transaction const & /*unused*/, auto i, auto n) {
[&] (store::read_transaction const & /*unused*/, auto i, auto n) {
auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::final_votes }));
for (; i != n; ++i)
{
auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::final_votes }));
rocksdb_transaction.refresh_if_needed ();
rocksdb_store->final_vote.put (rocksdb_transaction, i->first, i->second);
if (auto count_l = ++count; count_l % 500000 == 0)
{
logger.info (nano::log::type::ledger, "{} entries converted", count_l);
}
}
});
logger.info (nano::log::type::ledger, "Finished converting {} entries", count.load ());

logger.info (nano::log::type::ledger, "Finalizing migration...");
auto lmdb_transaction (store.tx_begin_read ());
auto version = store.version.get (lmdb_transaction);
auto rocksdb_transaction (rocksdb_store->tx_begin_write ());
Expand Down Expand Up @@ -1362,6 +1439,9 @@ bool nano::ledger::migrate_lmdb_to_rocksdb (std::filesystem::path const & data_p
{
error |= rocksdb_store->confirmation_height.get (rocksdb_transaction, account, confirmation_height_info);
}

logger.info (nano::log::type::ledger, "Migration completed. Make sure to enable RocksDb in the config file under [node.rocksdb]");
logger.info (nano::log::type::ledger, "After confirming correct node operation, the data.ldb file can be deleted if no longer required");
}
else
{
Expand Down
Loading