From 2c6462d0aabb0722604340a14cda0bb4a3c5e339 Mon Sep 17 00:00:00 2001 From: gr0vity Date: Tue, 2 Jul 2024 07:49:02 +0200 Subject: [PATCH] Squash merge PR #4647: migrate_database_lmdb_to_rocksdb improvements --- nano/lib/logging_enums.hpp | 1 + nano/node/cli.cpp | 7 +--- nano/secure/ledger.cpp | 83 ++++++++++++++++++++++++++++++++++---- 3 files changed, 77 insertions(+), 14 deletions(-) diff --git a/nano/lib/logging_enums.hpp b/nano/lib/logging_enums.hpp index b2b00b350c..b4b7486acf 100644 --- a/nano/lib/logging_enums.hpp +++ b/nano/lib/logging_enums.hpp @@ -66,6 +66,7 @@ enum class type opencl_work, upnp, rep_crawler, + ledger, lmdb, rocksdb, txn_tracker, diff --git a/nano/node/cli.cpp b/nano/node/cli.cpp index 2e1e4df4a4..7a6d4be5aa 100644 --- a/nano/node/cli.cpp +++ b/nano/node/cli.cpp @@ -483,7 +483,6 @@ std::error_code nano::handle_node_options (boost::program_options::variables_map auto error (false); if (!node.node->init_error ()) { - std::cout << "Migrating LMDB database to RocksDB, might take a while..." << std::endl; error = node.node->ledger.migrate_lmdb_to_rocksdb (data_path); } else @@ -491,11 +490,7 @@ std::error_code nano::handle_node_options (boost::program_options::variables_map error = true; } - if (!error) - { - std::cout << "Migration completed, after confirming it is correct the data.ldb file can be deleted if no longer required" << std::endl; - } - else + if (error) { std::cerr << "There was an error migrating" << std::endl; } diff --git a/nano/secure/ledger.cpp b/nano/secure/ledger.cpp index 2c5f915038..e6d8a5c2a0 100644 --- a/nano/secure/ledger.cpp +++ b/nano/secure/ledger.cpp @@ -1241,12 +1241,25 @@ uint64_t nano::ledger::pruning_action (secure::write_transaction & transaction_a // A precondition is that the store is an LMDB store bool nano::ledger::migrate_lmdb_to_rocksdb (std::filesystem::path const & data_path_a) const { + nano::logger logger; + nano::logger::initialize (nano::log_config::daemon_default (), data_path_a); + + logger.info (nano::log::type::ledger, "Migrating LMDB database to RocksDB. This will take a while..."); + + std::filesystem::space_info si = std::filesystem::space (data_path_a); + auto file_size = std::filesystem::file_size (data_path_a / "data.ldb"); + const auto estimated_required_space = file_size * 0.65; // RocksDb database size is approximately 65% of the lmdb size + + if (si.available < estimated_required_space) + { + logger.warn (nano::log::type::ledger, "You may not have enough available disk space. Estimated free space requirement is {} GB", estimated_required_space / 1024 / 1024 / 1024); + } + boost::system::error_code error_chmod; nano::set_secure_perm_directory (data_path_a, error_chmod); auto rockdb_data_path = data_path_a / "rocksdb"; std::filesystem::remove_all (rockdb_data_path); - nano::logger logger; auto error (false); // Open rocksdb database @@ -1256,8 +1269,10 @@ bool nano::ledger::migrate_lmdb_to_rocksdb (std::filesystem::path const & data_p if (!rocksdb_store->init_error ()) { + logger.info (nano::log::type::ledger, "Step 1 of 7: Converting blocks table"); + std::atomic count = 0; store.block.for_each_par ( - [&rocksdb_store] (store::read_transaction const & /*unused*/, auto i, auto n) { + [&rocksdb_store, &count, &logger] (store::read_transaction const & /*unused*/, auto i, auto n) { for (; i != n; ++i) { auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::blocks })); @@ -1269,63 +1284,112 @@ bool nano::ledger::migrate_lmdb_to_rocksdb (std::filesystem::path const & data_p i->second.sideband.serialize (stream, i->second.block->type ()); } rocksdb_store->block.raw_put (rocksdb_transaction, vector, i->first); + + if (auto count_l = ++count; count_l % 5000000 == 0) + { + logger.info (nano::log::type::ledger, "{} million blocks converted", count_l / 1000000); + } } }); + logger.info (nano::log::type::ledger, "Finished converting {} blocks", count.load ()); + logger.info (nano::log::type::ledger, "Step 2 of 7: Converting pending table"); + count = 0; store.pending.for_each_par ( - [&rocksdb_store] (store::read_transaction const & /*unused*/, auto i, auto n) { + [&rocksdb_store, &count, &logger] (store::read_transaction const & /*unused*/, auto i, auto n) { for (; i != n; ++i) { auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::pending })); rocksdb_store->pending.put (rocksdb_transaction, i->first, i->second); + if (auto count_l = ++count; count_l % 500000 == 0) + { + logger.info (nano::log::type::ledger, "{} entries converted", count_l); + } } }); + logger.info (nano::log::type::ledger, "Finished converting {} entries", count.load ()); + logger.info (nano::log::type::ledger, "Step 3 of 7: Converting confirmation_height table"); + count = 0; store.confirmation_height.for_each_par ( - [&rocksdb_store] (store::read_transaction const & /*unused*/, auto i, auto n) { + [&rocksdb_store, &count, &logger] (store::read_transaction const & /*unused*/, auto i, auto n) { for (; i != n; ++i) { auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::confirmation_height })); rocksdb_store->confirmation_height.put (rocksdb_transaction, i->first, i->second); + if (auto count_l = ++count; count_l % 500000 == 0) + { + logger.info (nano::log::type::ledger, "{} entries converted", count_l); + } } }); + logger.info (nano::log::type::ledger, "Finished converting {} entries", count.load ()); + logger.info (nano::log::type::ledger, "Step 4 of 7: Converting accounts table"); + count = 0; store.account.for_each_par ( - [&rocksdb_store] (store::read_transaction const & /*unused*/, auto i, auto n) { + [&rocksdb_store, &count, &logger] (store::read_transaction const & /*unused*/, auto i, auto n) { for (; i != n; ++i) { auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::accounts })); rocksdb_store->account.put (rocksdb_transaction, i->first, i->second); + if (auto count_l = ++count; count_l % 500000 == 0) + { + logger.info (nano::log::type::ledger, "{} entries converted", count_l); + } } }); + logger.info (nano::log::type::ledger, "Finished converting {} entries", count.load ()); + logger.info (nano::log::type::ledger, "Step 5 of 7: Converting rep_weights table"); + count = 0; store.rep_weight.for_each_par ( - [&rocksdb_store] (store::read_transaction const & /*unused*/, auto i, auto n) { + [&rocksdb_store, &count, &logger] (store::read_transaction const & /*unused*/, auto i, auto n) { for (; i != n; ++i) { auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::rep_weights })); rocksdb_store->rep_weight.put (rocksdb_transaction, i->first, i->second.number ()); + if (auto count_l = ++count; count_l % 500000 == 0) + { + logger.info (nano::log::type::ledger, "{} entries converted", count_l); + } } }); + logger.info (nano::log::type::ledger, "Finished converting {} entries", count.load ()); + logger.info (nano::log::type::ledger, "Step 6 of 7: Converting pruned table"); + count = 0; store.pruned.for_each_par ( - [&rocksdb_store] (store::read_transaction const & /*unused*/, auto i, auto n) { + [&rocksdb_store, &count, &logger] (store::read_transaction const & /*unused*/, auto i, auto n) { for (; i != n; ++i) { auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::pruned })); rocksdb_store->pruned.put (rocksdb_transaction, i->first); + if (auto count_l = ++count; count_l % 500000 == 0) + { + logger.info (nano::log::type::ledger, "{} entries converted", count_l); + } } }); + logger.info (nano::log::type::ledger, "Finished converting {} entries", count.load ()); + logger.info (nano::log::type::ledger, "Step 7 of 7: Converting final_votes table"); + count = 0; store.final_vote.for_each_par ( - [&rocksdb_store] (store::read_transaction const & /*unused*/, auto i, auto n) { + [&rocksdb_store, &count, &logger] (store::read_transaction const & /*unused*/, auto i, auto n) { for (; i != n; ++i) { auto rocksdb_transaction (rocksdb_store->tx_begin_write ({}, { nano::tables::final_votes })); rocksdb_store->final_vote.put (rocksdb_transaction, i->first, i->second); + if (auto count_l = ++count; count_l % 500000 == 0) + { + logger.info (nano::log::type::ledger, "{} entries converted", count_l); + } } }); + logger.info (nano::log::type::ledger, "Finished converting {} entries", count.load ()); + logger.info (nano::log::type::ledger, "Finalizing migration..."); auto lmdb_transaction (store.tx_begin_read ()); auto version = store.version.get (lmdb_transaction); auto rocksdb_transaction (rocksdb_store->tx_begin_write ()); @@ -1362,6 +1426,9 @@ bool nano::ledger::migrate_lmdb_to_rocksdb (std::filesystem::path const & data_p { error |= rocksdb_store->confirmation_height.get (rocksdb_transaction, account, confirmation_height_info); } + + logger.info (nano::log::type::ledger, "Migration completed. Make sure to enable RocksDb in the config file under [node.rocksdb]"); + logger.info (nano::log::type::ledger, "After confirming correct node operation, the data.ldb file can be deleted if no longer required"); } else {