Skip to content

Commit

Permalink
Squash merge PR nanocurrency#4647: migrate_database_lmdb_to_rocksdb i…
Browse files Browse the repository at this point in the history 
…mprovements
  • Loading branch information
gr0vity committed Jul 2, 2024
1 parent ea1c44f commit 2c6462d
Show file tree
Hide file tree
Showing 3 changed files with 77 additions and 14 deletions.
1 change: 1 addition & 0 deletions nano/lib/logging_enums.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ enum class type
opencl_work,
upnp,
rep_crawler,
ledger,
lmdb,
rocksdb,
txn_tracker,
Expand Down
7 changes: 1 addition & 6 deletions nano/node/cli.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -483,19 +483,14 @@ std::error_code nano::handle_node_options (boost::program_options::variables_map
auto error (false);
if (!node.node->init_error ())
{
std::cout << "Migrating LMDB database to RocksDB, might take a while..." << std::endl;
error = node.node->ledger.migrate_lmdb_to_rocksdb (data_path);
}
else
{
error = true;
}

if (!error)
{
std::cout << "Migration completed, after confirming it is correct the data.ldb file can be deleted if no longer required" << std::endl;
}
else
if (error)
{
std::cerr << "There was an error migrating" << std::endl;
}
Expand Down
83 changes: 75 additions & 8 deletions nano/secure/ledger.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1241,12 +1241,25 @@ uint64_t nano::ledger::pruning_action (secure::write_transaction & transaction_a
// A precondition is that the store is an LMDB store
bool nano::ledger::migrate_lmdb_to_rocksdb (std::filesystem::path const & data_path_a) const
{
nano::logger logger;
nano::logger::initialize (nano::log_config::daemon_default (), data_path_a);

logger.info (nano::log::type::ledger, "Migrating LMDB database to RocksDB. This will take a while...");

std::filesystem::space_info si = std::filesystem::space (data_path_a);
auto file_size = std::filesystem::file_size (data_path_a / "data.ldb");
const auto estimated_required_space = file_size * 0.65; // RocksDb database size is approximately 65% of the lmdb size

if (si.available < estimated_required_space)
{
logger.warn (nano::log::type::ledger, "You may not have enough available disk space. Estimated free space requirement is {} GB", estimated_required_space / 1024 / 1024 / 1024);
}

boost::system::error_code error_chmod;
nano::set_secure_perm_directory (data_path_a, error_chmod);
auto rockdb_data_path = data_path_a / "rocksdb";
std::filesystem::remove_all (rockdb_data_path);

nano::logger logger;
auto error (false);

// Open rocksdb database
Expand All @@ -1256,8 +1269,10 @@ bool nano::ledger::migrate_lmdb_to_rocksdb (std::filesystem::path const & data_p

if (!rocksdb_store->init_error ())
{
logger.info (nano::log::type::ledger, "Step 1 of 7: Converting blocks table");
std::atomic<std::size_t> count = 0;
store.block.for_each_par (
[&rocksdb_store] (store::read_transaction const & /*unused*/, auto i, auto n) {
[&rocksdb_store, &count, &logger] (store::read_transaction const & /*unused*/, auto i, auto n) {
for (; i != n; ++i)
{
auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::blocks }));
Expand All @@ -1269,63 +1284,112 @@ bool nano::ledger::migrate_lmdb_to_rocksdb (std::filesystem::path const & data_p
i->second.sideband.serialize (stream, i->second.block->type ());
}
rocksdb_store->block.raw_put (rocksdb_transaction, vector, i->first);

if (auto count_l = ++count; count_l % 5000000 == 0)
{
logger.info (nano::log::type::ledger, "{} million blocks converted", count_l / 1000000);
}
}
});

logger.info (nano::log::type::ledger, "Finished converting {} blocks", count.load ());
logger.info (nano::log::type::ledger, "Step 2 of 7: Converting pending table");
count = 0;
store.pending.for_each_par (
[&rocksdb_store] (store::read_transaction const & /*unused*/, auto i, auto n) {
[&rocksdb_store, &count, &logger] (store::read_transaction const & /*unused*/, auto i, auto n) {
for (; i != n; ++i)
{
auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::pending }));
rocksdb_store->pending.put (rocksdb_transaction, i->first, i->second);
if (auto count_l = ++count; count_l % 500000 == 0)
{
logger.info (nano::log::type::ledger, "{} entries converted", count_l);
}
}
});

logger.info (nano::log::type::ledger, "Finished converting {} entries", count.load ());
logger.info (nano::log::type::ledger, "Step 3 of 7: Converting confirmation_height table");
count = 0;
store.confirmation_height.for_each_par (
[&rocksdb_store] (store::read_transaction const & /*unused*/, auto i, auto n) {
[&rocksdb_store, &count, &logger] (store::read_transaction const & /*unused*/, auto i, auto n) {
for (; i != n; ++i)
{
auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::confirmation_height }));
rocksdb_store->confirmation_height.put (rocksdb_transaction, i->first, i->second);
if (auto count_l = ++count; count_l % 500000 == 0)
{
logger.info (nano::log::type::ledger, "{} entries converted", count_l);
}
}
});

logger.info (nano::log::type::ledger, "Finished converting {} entries", count.load ());
logger.info (nano::log::type::ledger, "Step 4 of 7: Converting accounts table");
count = 0;
store.account.for_each_par (
[&rocksdb_store] (store::read_transaction const & /*unused*/, auto i, auto n) {
[&rocksdb_store, &count, &logger] (store::read_transaction const & /*unused*/, auto i, auto n) {
for (; i != n; ++i)
{
auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::accounts }));
rocksdb_store->account.put (rocksdb_transaction, i->first, i->second);
if (auto count_l = ++count; count_l % 500000 == 0)
{
logger.info (nano::log::type::ledger, "{} entries converted", count_l);
}
}
});

logger.info (nano::log::type::ledger, "Finished converting {} entries", count.load ());
logger.info (nano::log::type::ledger, "Step 5 of 7: Converting rep_weights table");
count = 0;
store.rep_weight.for_each_par (
[&rocksdb_store] (store::read_transaction const & /*unused*/, auto i, auto n) {
[&rocksdb_store, &count, &logger] (store::read_transaction const & /*unused*/, auto i, auto n) {
for (; i != n; ++i)
{
auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::rep_weights }));
rocksdb_store->rep_weight.put (rocksdb_transaction, i->first, i->second.number ());
if (auto count_l = ++count; count_l % 500000 == 0)
{
logger.info (nano::log::type::ledger, "{} entries converted", count_l);
}
}
});

logger.info (nano::log::type::ledger, "Finished converting {} entries", count.load ());
logger.info (nano::log::type::ledger, "Step 6 of 7: Converting pruned table");
count = 0;
store.pruned.for_each_par (
[&rocksdb_store] (store::read_transaction const & /*unused*/, auto i, auto n) {
[&rocksdb_store, &count, &logger] (store::read_transaction const & /*unused*/, auto i, auto n) {
for (; i != n; ++i)
{
auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::pruned }));
rocksdb_store->pruned.put (rocksdb_transaction, i->first);
if (auto count_l = ++count; count_l % 500000 == 0)
{
logger.info (nano::log::type::ledger, "{} entries converted", count_l);
}
}
});

logger.info (nano::log::type::ledger, "Finished converting {} entries", count.load ());
logger.info (nano::log::type::ledger, "Step 7 of 7: Converting final_votes table");
count = 0;
store.final_vote.for_each_par (
[&rocksdb_store] (store::read_transaction const & /*unused*/, auto i, auto n) {
[&rocksdb_store, &count, &logger] (store::read_transaction const & /*unused*/, auto i, auto n) {
for (; i != n; ++i)
{
auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::final_votes }));
rocksdb_store->final_vote.put (rocksdb_transaction, i->first, i->second);
if (auto count_l = ++count; count_l % 500000 == 0)
{
logger.info (nano::log::type::ledger, "{} entries converted", count_l);
}
}
});
logger.info (nano::log::type::ledger, "Finished converting {} entries", count.load ());

logger.info (nano::log::type::ledger, "Finalizing migration...");
auto lmdb_transaction (store.tx_begin_read ());
auto version = store.version.get (lmdb_transaction);
auto rocksdb_transaction (rocksdb_store->tx_begin_write ());
Expand Down Expand Up @@ -1362,6 +1426,9 @@ bool nano::ledger::migrate_lmdb_to_rocksdb (std::filesystem::path const & data_p
{
error |= rocksdb_store->confirmation_height.get (rocksdb_transaction, account, confirmation_height_info);
}

logger.info (nano::log::type::ledger, "Migration completed. Make sure to enable RocksDb in the config file under [node.rocksdb]");
logger.info (nano::log::type::ledger, "After confirming correct node operation, the data.ldb file can be deleted if no longer required");
}
else
{
Expand Down

0 comments on commit 2c6462d

Please sign in to comment.