From cff51c4a219e5116a8d810d4838c63e67b94f416 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20W=C3=B3jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Sun, 8 Oct 2023 20:55:19 +0200 Subject: [PATCH] Checks to ensure transactions execute in the context of their originating stores (#4297) * Introduce `id_dispenser` class * Checks to ensure transactions execute in the context of their originating stores * Remove legacy `nano::wallets::split_if_needed` --- nano/core_test/wallet.cpp | 52 +++++------ nano/lib/CMakeLists.txt | 1 + nano/lib/id_dispenser.hpp | 62 +++++++++++++ nano/node/wallet.cpp | 134 ++++++--------------------- nano/node/wallet.hpp | 18 ++-- nano/rpc_test/rpc.cpp | 2 +- nano/store/lmdb/iterator.hpp | 11 +-- nano/store/lmdb/lmdb.cpp | 8 +- nano/store/lmdb/lmdb.hpp | 4 +- nano/store/lmdb/lmdb_env.cpp | 2 + nano/store/lmdb/lmdb_env.hpp | 7 ++ nano/store/lmdb/transaction.cpp | 2 + nano/store/lmdb/transaction_impl.hpp | 4 +- nano/store/transaction.cpp | 45 +++++++++ nano/store/transaction.hpp | 11 +++ 15 files changed, 209 insertions(+), 154 deletions(-) create mode 100644 nano/lib/id_dispenser.hpp diff --git a/nano/core_test/wallet.cpp b/nano/core_test/wallet.cpp index 6aad11bc15..3c835e209e 100644 --- a/nano/core_test/wallet.cpp +++ b/nano/core_test/wallet.cpp @@ -19,7 +19,7 @@ TEST (wallet, no_special_keys_accounts) ASSERT_FALSE (init); auto transaction (env.tx_begin_write ()); nano::kdf kdf{ nano::dev::network_params.kdf_work }; - nano::wallet_store wallet (init, kdf, transaction, nano::dev::genesis->account (), 1, "0"); + nano::wallet_store wallet (init, kdf, transaction, env, nano::dev::genesis->account (), 1, "0"); ASSERT_FALSE (init); nano::keypair key1; ASSERT_FALSE (wallet.exists (transaction, key1.pub)); @@ -40,7 +40,7 @@ TEST (wallet, no_key) ASSERT_FALSE (init); auto transaction (env.tx_begin_write ()); nano::kdf kdf{ nano::dev::network_params.kdf_work }; - nano::wallet_store wallet (init, kdf, transaction, nano::dev::genesis->account (), 1, "0"); + nano::wallet_store wallet (init, kdf, transaction, env, nano::dev::genesis->account (), 1, "0"); ASSERT_FALSE (init); nano::keypair key1; nano::raw_key prv1; @@ -55,7 +55,7 @@ TEST (wallet, fetch_locked) ASSERT_FALSE (init); auto transaction (env.tx_begin_write ()); nano::kdf kdf{ nano::dev::network_params.kdf_work }; - nano::wallet_store wallet (init, kdf, transaction, nano::dev::genesis->account (), 1, "0"); + nano::wallet_store wallet (init, kdf, transaction, env, nano::dev::genesis->account (), 1, "0"); ASSERT_TRUE (wallet.valid_password (transaction)); nano::keypair key1; ASSERT_EQ (key1.pub, wallet.insert_adhoc (transaction, key1.prv)); @@ -77,7 +77,7 @@ TEST (wallet, retrieval) ASSERT_FALSE (init); auto transaction (env.tx_begin_write ()); nano::kdf kdf{ nano::dev::network_params.kdf_work }; - nano::wallet_store wallet (init, kdf, transaction, nano::dev::genesis->account (), 1, "0"); + nano::wallet_store wallet (init, kdf, transaction, env, nano::dev::genesis->account (), 1, "0"); ASSERT_FALSE (init); nano::keypair key1; ASSERT_TRUE (wallet.valid_password (transaction)); @@ -99,7 +99,7 @@ TEST (wallet, empty_iteration) ASSERT_FALSE (init); auto transaction (env.tx_begin_write ()); nano::kdf kdf{ nano::dev::network_params.kdf_work }; - nano::wallet_store wallet (init, kdf, transaction, nano::dev::genesis->account (), 1, "0"); + nano::wallet_store wallet (init, kdf, transaction, env, nano::dev::genesis->account (), 1, "0"); ASSERT_FALSE (init); auto i (wallet.begin (transaction)); auto j (wallet.end ()); @@ -113,7 +113,7 @@ TEST (wallet, one_item_iteration) ASSERT_FALSE (init); auto transaction (env.tx_begin_write ()); nano::kdf kdf{ nano::dev::network_params.kdf_work }; - nano::wallet_store wallet (init, kdf, transaction, nano::dev::genesis->account (), 1, "0"); + nano::wallet_store wallet (init, kdf, transaction, env, nano::dev::genesis->account (), 1, "0"); ASSERT_FALSE (init); nano::keypair key1; wallet.insert_adhoc (transaction, key1.prv); @@ -141,7 +141,7 @@ TEST (wallet, two_item_iteration) nano::kdf kdf{ nano::dev::network_params.kdf_work }; { auto transaction (env.tx_begin_write ()); - nano::wallet_store wallet (init, kdf, transaction, nano::dev::genesis->account (), 1, "0"); + nano::wallet_store wallet (init, kdf, transaction, env, nano::dev::genesis->account (), 1, "0"); ASSERT_FALSE (init); wallet.insert_adhoc (transaction, key1.prv); wallet.insert_adhoc (transaction, key2.prv); @@ -274,7 +274,7 @@ TEST (wallet, find_none) ASSERT_FALSE (init); auto transaction (env.tx_begin_write ()); nano::kdf kdf{ nano::dev::network_params.kdf_work }; - nano::wallet_store wallet (init, kdf, transaction, nano::dev::genesis->account (), 1, "0"); + nano::wallet_store wallet (init, kdf, transaction, env, nano::dev::genesis->account (), 1, "0"); ASSERT_FALSE (init); nano::account account (1000); ASSERT_EQ (wallet.end (), wallet.find (transaction, account)); @@ -287,7 +287,7 @@ TEST (wallet, find_existing) ASSERT_FALSE (init); auto transaction (env.tx_begin_write ()); nano::kdf kdf{ nano::dev::network_params.kdf_work }; - nano::wallet_store wallet (init, kdf, transaction, nano::dev::genesis->account (), 1, "0"); + nano::wallet_store wallet (init, kdf, transaction, env, nano::dev::genesis->account (), 1, "0"); ASSERT_FALSE (init); nano::keypair key1; ASSERT_FALSE (wallet.exists (transaction, key1.pub)); @@ -306,7 +306,7 @@ TEST (wallet, rekey) ASSERT_FALSE (init); auto transaction (env.tx_begin_write ()); nano::kdf kdf{ nano::dev::network_params.kdf_work }; - nano::wallet_store wallet (init, kdf, transaction, nano::dev::genesis->account (), 1, "0"); + nano::wallet_store wallet (init, kdf, transaction, env, nano::dev::genesis->account (), 1, "0"); ASSERT_FALSE (init); nano::raw_key password; wallet.password.value (password); @@ -378,7 +378,7 @@ TEST (wallet, hash_password) ASSERT_FALSE (init); auto transaction (env.tx_begin_write ()); nano::kdf kdf{ nano::dev::network_params.kdf_work }; - nano::wallet_store wallet (init, kdf, transaction, nano::dev::genesis->account (), 1, "0"); + nano::wallet_store wallet (init, kdf, transaction, env, nano::dev::genesis->account (), 1, "0"); ASSERT_FALSE (init); nano::raw_key hash1; wallet.derive_key (hash1, transaction, ""); @@ -428,25 +428,25 @@ TEST (wallet, reopen_default_password) ASSERT_FALSE (init); nano::kdf kdf{ nano::dev::network_params.kdf_work }; { - nano::wallet_store wallet (init, kdf, transaction, nano::dev::genesis->account (), 1, "0"); + nano::wallet_store wallet (init, kdf, transaction, env, nano::dev::genesis->account (), 1, "0"); ASSERT_FALSE (init); ASSERT_TRUE (wallet.valid_password (transaction)); } { bool init; - nano::wallet_store wallet (init, kdf, transaction, nano::dev::genesis->account (), 1, "0"); + nano::wallet_store wallet (init, kdf, transaction, env, nano::dev::genesis->account (), 1, "0"); ASSERT_FALSE (init); ASSERT_TRUE (wallet.valid_password (transaction)); } { - nano::wallet_store wallet (init, kdf, transaction, nano::dev::genesis->account (), 1, "0"); + nano::wallet_store wallet (init, kdf, transaction, env, nano::dev::genesis->account (), 1, "0"); ASSERT_FALSE (init); wallet.rekey (transaction, ""); ASSERT_TRUE (wallet.valid_password (transaction)); } { bool init; - nano::wallet_store wallet (init, kdf, transaction, nano::dev::genesis->account (), 1, "0"); + nano::wallet_store wallet (init, kdf, transaction, env, nano::dev::genesis->account (), 1, "0"); ASSERT_FALSE (init); ASSERT_FALSE (wallet.valid_password (transaction)); wallet.attempt_password (transaction, " "); @@ -463,7 +463,7 @@ TEST (wallet, representative) ASSERT_FALSE (error); auto transaction (env.tx_begin_write ()); nano::kdf kdf{ nano::dev::network_params.kdf_work }; - nano::wallet_store wallet (error, kdf, transaction, nano::dev::genesis->account (), 1, "0"); + nano::wallet_store wallet (error, kdf, transaction, env, nano::dev::genesis->account (), 1, "0"); ASSERT_FALSE (error); ASSERT_FALSE (wallet.is_representative (transaction)); ASSERT_EQ (nano::dev::genesis->account (), wallet.representative (transaction)); @@ -484,11 +484,11 @@ TEST (wallet, serialize_json_empty) ASSERT_FALSE (error); auto transaction (env.tx_begin_write ()); nano::kdf kdf{ nano::dev::network_params.kdf_work }; - nano::wallet_store wallet1 (error, kdf, transaction, nano::dev::genesis->account (), 1, "0"); + nano::wallet_store wallet1 (error, kdf, transaction, env, nano::dev::genesis->account (), 1, "0"); ASSERT_FALSE (error); std::string serialized; wallet1.serialize_json (transaction, serialized); - nano::wallet_store wallet2 (error, kdf, transaction, nano::dev::genesis->account (), 1, "1", serialized); + nano::wallet_store wallet2 (error, kdf, transaction, env, nano::dev::genesis->account (), 1, "1", serialized); ASSERT_FALSE (error); nano::raw_key password1; nano::raw_key password2; @@ -509,13 +509,13 @@ TEST (wallet, serialize_json_one) ASSERT_FALSE (error); auto transaction (env.tx_begin_write ()); nano::kdf kdf{ nano::dev::network_params.kdf_work }; - nano::wallet_store wallet1 (error, kdf, transaction, nano::dev::genesis->account (), 1, "0"); + nano::wallet_store wallet1 (error, kdf, transaction, env, nano::dev::genesis->account (), 1, "0"); ASSERT_FALSE (error); nano::keypair key; wallet1.insert_adhoc (transaction, key.prv); std::string serialized; wallet1.serialize_json (transaction, serialized); - nano::wallet_store wallet2 (error, kdf, transaction, nano::dev::genesis->account (), 1, "1", serialized); + nano::wallet_store wallet2 (error, kdf, transaction, env, nano::dev::genesis->account (), 1, "1", serialized); ASSERT_FALSE (error); nano::raw_key password1; nano::raw_key password2; @@ -538,14 +538,14 @@ TEST (wallet, serialize_json_password) ASSERT_FALSE (error); auto transaction (env.tx_begin_write ()); nano::kdf kdf{ nano::dev::network_params.kdf_work }; - nano::wallet_store wallet1 (error, kdf, transaction, nano::dev::genesis->account (), 1, "0"); + nano::wallet_store wallet1 (error, kdf, transaction, env, nano::dev::genesis->account (), 1, "0"); ASSERT_FALSE (error); nano::keypair key; wallet1.rekey (transaction, "password"); wallet1.insert_adhoc (transaction, key.prv); std::string serialized; wallet1.serialize_json (transaction, serialized); - nano::wallet_store wallet2 (error, kdf, transaction, nano::dev::genesis->account (), 1, "1", serialized); + nano::wallet_store wallet2 (error, kdf, transaction, env, nano::dev::genesis->account (), 1, "1", serialized); ASSERT_FALSE (error); ASSERT_FALSE (wallet2.valid_password (transaction)); ASSERT_FALSE (wallet2.attempt_password (transaction, "password")); @@ -571,11 +571,11 @@ TEST (wallet_store, move) ASSERT_FALSE (error); auto transaction (env.tx_begin_write ()); nano::kdf kdf{ nano::dev::network_params.kdf_work }; - nano::wallet_store wallet1 (error, kdf, transaction, nano::dev::genesis->account (), 1, "0"); + nano::wallet_store wallet1 (error, kdf, transaction, env, nano::dev::genesis->account (), 1, "0"); ASSERT_FALSE (error); nano::keypair key1; wallet1.insert_adhoc (transaction, key1.prv); - nano::wallet_store wallet2 (error, kdf, transaction, nano::dev::genesis->account (), 1, "1"); + nano::wallet_store wallet2 (error, kdf, transaction, env, nano::dev::genesis->account (), 1, "1"); ASSERT_FALSE (error); nano::keypair key2; wallet2.insert_adhoc (transaction, key2.prv); @@ -729,7 +729,7 @@ TEST (wallet, deterministic_keys) ASSERT_FALSE (init); auto transaction (env.tx_begin_write ()); nano::kdf kdf{ nano::dev::network_params.kdf_work }; - nano::wallet_store wallet (init, kdf, transaction, nano::dev::genesis->account (), 1, "0"); + nano::wallet_store wallet (init, kdf, transaction, env, nano::dev::genesis->account (), 1, "0"); auto key1 = wallet.deterministic_key (transaction, 0); auto key2 = wallet.deterministic_key (transaction, 0); ASSERT_EQ (key1, key2); @@ -772,7 +772,7 @@ TEST (wallet, reseed) ASSERT_FALSE (init); auto transaction (env.tx_begin_write ()); nano::kdf kdf{ nano::dev::network_params.kdf_work }; - nano::wallet_store wallet (init, kdf, transaction, nano::dev::genesis->account (), 1, "0"); + nano::wallet_store wallet (init, kdf, transaction, env, nano::dev::genesis->account (), 1, "0"); nano::raw_key seed1; seed1 = 1; nano::raw_key seed2; diff --git a/nano/lib/CMakeLists.txt b/nano/lib/CMakeLists.txt index 79d348f976..934cd1452a 100644 --- a/nano/lib/CMakeLists.txt +++ b/nano/lib/CMakeLists.txt @@ -37,6 +37,7 @@ add_library( epoch.cpp errors.hpp errors.cpp + id_dispenser.hpp ipc.hpp ipc.cpp ipc_client.hpp diff --git a/nano/lib/id_dispenser.hpp b/nano/lib/id_dispenser.hpp new file mode 100644 index 0000000000..4b0cc53fe5 --- /dev/null +++ b/nano/lib/id_dispenser.hpp @@ -0,0 +1,62 @@ +#pragma once + +#include +#include + +namespace nano +{ +class id_dispenser +{ +public: + enum class mode + { + sequential, + random, + }; + + // Using pointer type for prettier and more concise output in logs (hex) + using id_t = void *; + +public: + explicit id_dispenser (mode mode = mode::random) : + mode_m{ mode } + { + } + + id_t next_id () + { + switch (mode_m) + { + case mode::sequential: + return reinterpret_cast (current_id_m.fetch_add (1)); + case mode::random: + auto value = get_dist () (get_rng ()); + if (value < min_m) + { + value += min_m; + } + return reinterpret_cast (value); + } + return 0; + } + +private: + // Avoid IDs with leading 0s for nicer output in logs + static constexpr uint64_t min_m{ 0x1000000000000000 }; + + mode mode_m; + std::atomic current_id_m{ min_m }; + + static std::mt19937 & get_rng () + { + static thread_local std::mt19937 rng{ std::random_device{}() }; + return rng; + } + + static std::uniform_int_distribution & get_dist () + { + static thread_local std::uniform_int_distribution dist; + return dist; + } +}; +} \ No newline at end of file diff --git a/nano/node/wallet.cpp b/nano/node/wallet.cpp index 8f2ec7e0ca..cba2ed2b14 100644 --- a/nano/node/wallet.cpp +++ b/nano/node/wallet.cpp @@ -255,17 +255,18 @@ int const nano::wallet_store::special_count (7); std::size_t const nano::wallet_store::check_iv_index (0); std::size_t const nano::wallet_store::seed_iv_index (1); -nano::wallet_store::wallet_store (bool & init_a, nano::kdf & kdf_a, store::transaction & transaction_a, nano::account representative_a, unsigned fanout_a, std::string const & wallet_a, std::string const & json_a) : +nano::wallet_store::wallet_store (bool & init_a, nano::kdf & kdf_a, store::transaction & transaction_a, store::lmdb::env & env_a, nano::account representative_a, unsigned fanout_a, std::string const & wallet_a, std::string const & json_a) : password (0, fanout_a), wallet_key_mem (0, fanout_a), - kdf (kdf_a) + kdf (kdf_a), + env{ env_a } { init_a = false; initialize (transaction_a, init_a, wallet_a); if (!init_a) { MDB_val junk; - debug_assert (mdb_get (tx (transaction_a), handle, nano::store::lmdb::db_val (version_special), &junk) == MDB_NOTFOUND); + debug_assert (mdb_get (env.tx (transaction_a), handle, nano::store::lmdb::db_val (version_special), &junk) == MDB_NOTFOUND); boost::property_tree::ptree wallet_l; std::stringstream istream (json_a); try @@ -298,11 +299,11 @@ nano::wallet_store::wallet_store (bool & init_a, nano::kdf & kdf_a, store::trans init_a = true; } } - init_a |= mdb_get (tx (transaction_a), handle, nano::store::lmdb::db_val (version_special), &junk) != 0; - init_a |= mdb_get (tx (transaction_a), handle, nano::store::lmdb::db_val (wallet_key_special), &junk) != 0; - init_a |= mdb_get (tx (transaction_a), handle, nano::store::lmdb::db_val (salt_special), &junk) != 0; - init_a |= mdb_get (tx (transaction_a), handle, nano::store::lmdb::db_val (check_special), &junk) != 0; - init_a |= mdb_get (tx (transaction_a), handle, nano::store::lmdb::db_val (representative_special), &junk) != 0; + init_a |= mdb_get (env.tx (transaction_a), handle, nano::store::lmdb::db_val (version_special), &junk) != 0; + init_a |= mdb_get (env.tx (transaction_a), handle, nano::store::lmdb::db_val (wallet_key_special), &junk) != 0; + init_a |= mdb_get (env.tx (transaction_a), handle, nano::store::lmdb::db_val (salt_special), &junk) != 0; + init_a |= mdb_get (env.tx (transaction_a), handle, nano::store::lmdb::db_val (check_special), &junk) != 0; + init_a |= mdb_get (env.tx (transaction_a), handle, nano::store::lmdb::db_val (representative_special), &junk) != 0; nano::raw_key key; key.clear (); password.value_set (key); @@ -311,10 +312,11 @@ nano::wallet_store::wallet_store (bool & init_a, nano::kdf & kdf_a, store::trans } } -nano::wallet_store::wallet_store (bool & init_a, nano::kdf & kdf_a, store::transaction & transaction_a, nano::account representative_a, unsigned fanout_a, std::string const & wallet_a) : +nano::wallet_store::wallet_store (bool & init_a, nano::kdf & kdf_a, store::transaction & transaction_a, store::lmdb::env & env_a, nano::account representative_a, unsigned fanout_a, std::string const & wallet_a) : password (0, fanout_a), wallet_key_mem (0, fanout_a), - kdf (kdf_a) + kdf (kdf_a), + env{ env_a } { init_a = false; initialize (transaction_a, init_a, wallet_a); @@ -322,7 +324,7 @@ nano::wallet_store::wallet_store (bool & init_a, nano::kdf & kdf_a, store::trans { int version_status; MDB_val version_value; - version_status = mdb_get (tx (transaction_a), handle, nano::store::lmdb::db_val (version_special), &version_value); + version_status = mdb_get (env.tx (transaction_a), handle, nano::store::lmdb::db_val (version_special), &version_value); if (version_status == MDB_NOTFOUND) { version_put (transaction_a, version_current); @@ -377,7 +379,7 @@ void nano::wallet_store::initialize (store::transaction const & transaction_a, b debug_assert (strlen (path_a.c_str ()) == path_a.size ()); auto error (0); MDB_dbi handle_l; - error |= mdb_dbi_open (tx (transaction_a), path_a.c_str (), MDB_CREATE, &handle_l); + error |= mdb_dbi_open (env.tx (transaction_a), path_a.c_str (), MDB_CREATE, &handle_l); handle = handle_l; init_a = error != 0; } @@ -424,7 +426,7 @@ bool nano::wallet_store::insert_watch (store::transaction const & transaction_a, void nano::wallet_store::erase (store::transaction const & transaction_a, nano::account const & pub) { - auto status (mdb_del (tx (transaction_a), handle, nano::store::lmdb::db_val (pub), nullptr)); + auto status (mdb_del (env.tx (transaction_a), handle, nano::store::lmdb::db_val (pub), nullptr)); (void)status; debug_assert (status == 0); } @@ -433,7 +435,7 @@ nano::wallet_value nano::wallet_store::entry_get_raw (store::transaction const & { nano::wallet_value result; nano::store::lmdb::db_val value; - auto status (mdb_get (tx (transaction_a), handle, nano::store::lmdb::db_val (pub_a), value)); + auto status (mdb_get (env.tx (transaction_a), handle, nano::store::lmdb::db_val (pub_a), value)); if (status == 0) { result = nano::wallet_value (value); @@ -448,7 +450,7 @@ nano::wallet_value nano::wallet_store::entry_get_raw (store::transaction const & void nano::wallet_store::entry_put_raw (store::transaction const & transaction_a, nano::account const & pub_a, nano::wallet_value const & entry_a) { - auto status (mdb_put (tx (transaction_a), handle, nano::store::lmdb::db_val (pub_a), nano::store::lmdb::db_val (sizeof (entry_a), const_cast (&entry_a)), 0)); + auto status (mdb_put (env.tx (transaction_a), handle, nano::store::lmdb::db_val (pub_a), nano::store::lmdb::db_val (sizeof (entry_a), const_cast (&entry_a)), 0)); (void)status; debug_assert (status == 0); } @@ -542,7 +544,7 @@ bool nano::wallet_store::exists (store::transaction const & transaction_a, nano: void nano::wallet_store::serialize_json (store::transaction const & transaction_a, std::string & string_a) { boost::property_tree::ptree tree; - for (store::iterator i (std::make_unique> (transaction_a, handle)), n (nullptr); i != n; ++i) + for (store::iterator i (std::make_unique> (transaction_a, env, handle)), n (nullptr); i != n; ++i) { tree.put (i->first.to_string (), i->second.key.to_string ()); } @@ -659,14 +661,14 @@ void nano::kdf::phs (nano::raw_key & result_a, std::string const & password_a, n nano::wallet::wallet (bool & init_a, store::transaction & transaction_a, nano::wallets & wallets_a, std::string const & wallet_a) : lock_observer ([] (bool, bool) {}), - store (init_a, wallets_a.kdf, transaction_a, wallets_a.node.config.random_representative (), wallets_a.node.config.password_fanout, wallet_a), + store (init_a, wallets_a.kdf, transaction_a, wallets_a.env, wallets_a.node.config.random_representative (), wallets_a.node.config.password_fanout, wallet_a), wallets (wallets_a) { } nano::wallet::wallet (bool & init_a, store::transaction & transaction_a, nano::wallets & wallets_a, std::string const & wallet_a, std::string const & json) : lock_observer ([] (bool, bool) {}), - store (init_a, wallets_a.kdf, transaction_a, wallets_a.node.config.random_representative (), wallets_a.node.config.password_fanout, wallet_a, json), + store (init_a, wallets_a.kdf, transaction_a, wallets_a.env, wallets_a.node.config.random_representative (), wallets_a.node.config.password_fanout, wallet_a, json), wallets (wallets_a) { } @@ -798,7 +800,7 @@ bool nano::wallet::import (std::string const & json_a, std::string const & passw auto transaction (wallets.tx_begin_write ()); nano::uint256_union id; random_pool::generate_block (id.bytes.data (), id.bytes.size ()); - temp = std::make_unique (error, wallets.node.wallets.kdf, transaction, 0, 1, id.to_string (), json_a); + temp = std::make_unique (error, wallets.node.wallets.kdf, transaction, wallets.env, 0, 1, id.to_string (), json_a); } if (!error) { @@ -822,7 +824,7 @@ void nano::wallet::serialize (std::string & json_a) void nano::wallet_store::destroy (store::transaction const & transaction_a) { - auto status (mdb_drop (tx (transaction_a), handle, 1)); + auto status (mdb_drop (env.tx (transaction_a), handle, 1)); (void)status; debug_assert (status == 0); handle = 0; @@ -1343,13 +1345,12 @@ nano::wallets::wallets (bool error_a, nano::node & node_a) : { auto transaction (tx_begin_write ()); auto status (mdb_dbi_open (env.tx (transaction), nullptr, MDB_CREATE, &handle)); - split_if_needed (transaction, node.store); status |= mdb_dbi_open (env.tx (transaction), "send_action_ids", MDB_CREATE, &send_action_ids); release_assert (status == 0); std::string beginning (nano::uint256_union (0).to_string ()); std::string end ((nano::uint256_union (nano::uint256_t (0) - nano::uint256_t (1))).to_string ()); - store::iterator, nano::no_value> i (std::make_unique, nano::no_value>> (transaction, handle, nano::store::lmdb::db_val (beginning.size (), const_cast (beginning.c_str ())))); - store::iterator, nano::no_value> n (std::make_unique, nano::no_value>> (transaction, handle, nano::store::lmdb::db_val (end.size (), const_cast (end.c_str ())))); + store::iterator, nano::no_value> i (std::make_unique, nano::no_value>> (transaction, env, handle, nano::store::lmdb::db_val (beginning.size (), const_cast (beginning.c_str ())))); + store::iterator, nano::no_value> n (std::make_unique, nano::no_value>> (transaction, env, handle, nano::store::lmdb::db_val (end.size (), const_cast (end.c_str ())))); for (; i != n; ++i) { nano::wallet_id id; @@ -1477,8 +1478,8 @@ void nano::wallets::reload () std::unordered_set stored_items; std::string beginning (nano::uint256_union (0).to_string ()); std::string end ((nano::uint256_union (nano::uint256_t (0) - nano::uint256_t (1))).to_string ()); - store::iterator, nano::no_value> i (std::make_unique, nano::no_value>> (transaction, handle, nano::store::lmdb::db_val (beginning.size (), const_cast (beginning.c_str ())))); - store::iterator, nano::no_value> n (std::make_unique, nano::no_value>> (transaction, handle, nano::store::lmdb::db_val (end.size (), const_cast (end.c_str ())))); + store::iterator, nano::no_value> i (std::make_unique, nano::no_value>> (transaction, env, handle, nano::store::lmdb::db_val (beginning.size (), const_cast (beginning.c_str ())))); + store::iterator, nano::no_value> n (std::make_unique, nano::no_value>> (transaction, env, handle, nano::store::lmdb::db_val (end.size (), const_cast (end.c_str ())))); for (; i != n; ++i) { nano::wallet_id id; @@ -1696,80 +1697,6 @@ void nano::wallets::ongoing_compute_reps () }); } -void nano::wallets::split_if_needed (store::transaction & transaction_destination, nano::store::component & store_a) -{ - auto store_l = dynamic_cast (&store_a); - if (store_l != nullptr) - { - if (items.empty ()) - { - std::string beginning (nano::uint256_union (0).to_string ()); - std::string end ((nano::uint256_union (nano::uint256_t (0) - nano::uint256_t (1))).to_string ()); - - auto get_store_it = [&handle = handle] (store::transaction const & transaction_source, std::string const & hash) { - return store::iterator, nano::no_value> (std::make_unique, nano::no_value>> (transaction_source, handle, nano::store::lmdb::db_val (hash.size (), const_cast (hash.c_str ())))); - }; - - // First do a read pass to check if there are any wallets that need extracting (to save holding a write lock and potentially being blocked) - auto wallets_need_splitting (false); - { - auto transaction_source (store_l->tx_begin_read ()); - auto i = get_store_it (transaction_source, beginning); - auto n = get_store_it (transaction_source, end); - wallets_need_splitting = (i != n); - } - - if (wallets_need_splitting) - { - auto transaction_source (store_l->tx_begin_write ()); - auto i = get_store_it (transaction_source, beginning); - auto n = get_store_it (transaction_source, end); - auto tx_source = static_cast (transaction_source.get_handle ()); - auto tx_destination = static_cast (transaction_destination.get_handle ()); - for (; i != n; ++i) - { - nano::uint256_union id; - std::string text (i->first.data (), i->first.size ()); - auto error1 (id.decode_hex (text)); - (void)error1; - debug_assert (!error1); - debug_assert (strlen (text.c_str ()) == text.size ()); - move_table (text, tx_source, tx_destination); - } - } - } - } -} - -void nano::wallets::move_table (std::string const & name_a, MDB_txn * tx_source, MDB_txn * tx_destination) -{ - MDB_dbi handle_source; - auto error2 (mdb_dbi_open (tx_source, name_a.c_str (), MDB_CREATE, &handle_source)); - (void)error2; - debug_assert (!error2); - MDB_dbi handle_destination; - auto error3 (mdb_dbi_open (tx_destination, name_a.c_str (), MDB_CREATE, &handle_destination)); - (void)error3; - debug_assert (!error3); - MDB_cursor * cursor; - auto error4 (mdb_cursor_open (tx_source, handle_source, &cursor)); - (void)error4; - debug_assert (!error4); - MDB_val val_key; - MDB_val val_value; - auto cursor_status (mdb_cursor_get (cursor, &val_key, &val_value, MDB_FIRST)); - while (cursor_status == MDB_SUCCESS) - { - auto error5 (mdb_put (tx_destination, handle_destination, &val_key, &val_value, 0)); - (void)error5; - debug_assert (!error5); - cursor_status = mdb_cursor_get (cursor, &val_key, &val_value, MDB_NEXT); - } - auto error6 (mdb_drop (tx_source, handle_source, 1)); - (void)error6; - debug_assert (!error6); -} - std::unordered_map> nano::wallets::get_wallets () { debug_assert (!mutex.try_lock ()); @@ -1781,13 +1708,13 @@ nano::uint128_t const nano::wallets::high_priority = std::numeric_limits nano::wallet_store::begin (store::transaction const & transaction_a) { - store::iterator result (std::make_unique> (transaction_a, handle, nano::store::lmdb::db_val (nano::account (special_count)))); + store::iterator result (std::make_unique> (transaction_a, env, handle, nano::store::lmdb::db_val (nano::account (special_count)))); return result; } nano::store::iterator nano::wallet_store::begin (store::transaction const & transaction_a, nano::account const & key) { - store::iterator result (std::make_unique> (transaction_a, handle, nano::store::lmdb::db_val (key))); + store::iterator result (std::make_unique> (transaction_a, env, handle, nano::store::lmdb::db_val (key))); return result; } @@ -1827,11 +1754,6 @@ bool nano::mdb_wallets_store::init_error () const return error; } -MDB_txn * nano::wallet_store::tx (store::transaction const & transaction_a) const -{ - return static_cast (transaction_a.get_handle ()); -} - std::unique_ptr nano::collect_container_info (wallets & wallets, std::string const & name) { std::size_t items_count; diff --git a/nano/node/wallet.hpp b/nano/node/wallet.hpp index c585ae2995..12d73ce09f 100644 --- a/nano/node/wallet.hpp +++ b/nano/node/wallet.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -13,11 +14,13 @@ #include #include #include + namespace nano { class node; class node_config; class wallets; + // The fan spreads a key out over the heap to decrease the likelihood of it being recovered by memory inspection class fan final { @@ -31,6 +34,7 @@ class fan final nano::mutex mutex; void value_get (nano::raw_key &); }; + class kdf final { public: @@ -42,6 +46,7 @@ class kdf final nano::mutex mutex; unsigned & kdf_work; }; + enum class key_type { not_a_type, @@ -49,11 +54,12 @@ enum class key_type adhoc, deterministic }; + class wallet_store final { public: - wallet_store (bool &, nano::kdf &, store::transaction &, nano::account, unsigned, std::string const &); - wallet_store (bool &, nano::kdf &, store::transaction &, nano::account, unsigned, std::string const &, std::string const &); + wallet_store (bool &, nano::kdf &, store::transaction &, store::lmdb::env &, nano::account, unsigned, std::string const &); + wallet_store (bool &, nano::kdf &, store::transaction &, store::lmdb::env &, nano::account, unsigned, std::string const &, std::string const &); std::vector accounts (store::transaction const &); void initialize (store::transaction const &, bool &, std::string const &); nano::uint256_union check (store::transaction const &); @@ -118,8 +124,9 @@ class wallet_store final std::recursive_mutex mutex; private: - MDB_txn * tx (store::transaction const &) const; + nano::store::lmdb::env & env; }; + // A wallet is a set of account keys encrypted by a common encryption key class wallet final : public std::enable_shared_from_this { @@ -213,8 +220,6 @@ class wallets final bool check_rep (nano::account const &, nano::uint128_t const &, bool const = true); void compute_reps (); void ongoing_compute_reps (); - void split_if_needed (store::transaction &, nano::store::component &); - void move_table (std::string const &, MDB_txn *, MDB_txn *); std::unordered_map> get_wallets (); nano::network_params & network_params; std::function observer; @@ -233,9 +238,9 @@ class wallets final std::thread thread; static nano::uint128_t const generate_priority; static nano::uint128_t const high_priority; + /** Start read-write transaction */ store::write_transaction tx_begin_write (); - /** Start read-only transaction */ store::read_transaction tx_begin_read (); @@ -252,6 +257,7 @@ class wallets_store virtual ~wallets_store () = default; virtual bool init_error () const = 0; }; + class mdb_wallets_store final : public wallets_store { public: diff --git a/nano/rpc_test/rpc.cpp b/nano/rpc_test/rpc.cpp index 59b83a9fe0..615d672928 100644 --- a/nano/rpc_test/rpc.cpp +++ b/nano/rpc_test/rpc.cpp @@ -660,7 +660,7 @@ TEST (rpc, wallet_export) bool error (false); auto transaction (node->wallets.tx_begin_write ()); nano::kdf kdf{ nano::dev::network_params.kdf_work }; - nano::wallet_store store (error, kdf, transaction, nano::dev::genesis->account (), 1, "0", wallet_json); + nano::wallet_store store (error, kdf, transaction, node->wallets.env, nano::dev::genesis->account (), 1, "0", wallet_json); ASSERT_FALSE (error); ASSERT_TRUE (store.exists (transaction, nano::dev::genesis_key.pub)); } diff --git a/nano/store/lmdb/iterator.hpp b/nano/store/lmdb/iterator.hpp index 9a9addf3ff..e8230fd73f 100644 --- a/nano/store/lmdb/iterator.hpp +++ b/nano/store/lmdb/iterator.hpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -13,9 +14,9 @@ template class iterator : public iterator_impl { public: - iterator (store::transaction const & transaction_a, MDB_dbi db_a, MDB_val const & val_a = MDB_val{}, bool const direction_asc = true) + iterator (store::transaction const & transaction_a, env const & env_a, MDB_dbi db_a, MDB_val const & val_a = MDB_val{}, bool const direction_asc = true) { - auto status (mdb_cursor_open (tx (transaction_a), db_a, &cursor)); + auto status (mdb_cursor_open (env_a.tx (transaction_a), db_a, &cursor)); release_assert (status == 0); auto operation (MDB_SET_RANGE); if (val_a.mv_size != 0) @@ -165,12 +166,6 @@ class iterator : public iterator_impl store::iterator_impl & operator= (store::iterator_impl const &) = delete; MDB_cursor * cursor{ nullptr }; std::pair, store::db_val> current; - -private: - MDB_txn * tx (store::transaction const & transaction_a) const - { - return static_cast (transaction_a.get_handle ()); - } }; /** diff --git a/nano/store/lmdb/lmdb.cpp b/nano/store/lmdb/lmdb.cpp index e1385ac6da..811fe1ecea 100644 --- a/nano/store/lmdb/lmdb.cpp +++ b/nano/store/lmdb/lmdb.cpp @@ -372,7 +372,7 @@ void nano::store::lmdb::component::rebuild_db (store::write_transaction const & MDB_dbi temp; mdb_dbi_open (env.tx (transaction_a), "temp_table", MDB_CREATE, &temp); // Copy all values to temporary table - for (auto i (store::iterator (std::make_unique> (transaction_a, table))), n (store::iterator (nullptr)); i != n; ++i) + for (auto i (store::iterator (std::make_unique> (transaction_a, env, table))), n (store::iterator (nullptr)); i != n; ++i) { auto s = mdb_put (env.tx (transaction_a), temp, nano::store::lmdb::db_val (i->first), i->second, MDB_APPEND); release_assert_success (s); @@ -381,7 +381,7 @@ void nano::store::lmdb::component::rebuild_db (store::write_transaction const & // Clear existing table mdb_drop (env.tx (transaction_a), table, 0); // Put values from copy - for (auto i (store::iterator (std::make_unique> (transaction_a, temp))), n (store::iterator (nullptr)); i != n; ++i) + for (auto i (store::iterator (std::make_unique> (transaction_a, env, temp))), n (store::iterator (nullptr)); i != n; ++i) { auto s = mdb_put (env.tx (transaction_a), table, nano::store::lmdb::db_val (i->first), i->second, MDB_APPEND); release_assert_success (s); @@ -395,7 +395,7 @@ void nano::store::lmdb::component::rebuild_db (store::write_transaction const & MDB_dbi temp; mdb_dbi_open (env.tx (transaction_a), "temp_table", MDB_CREATE, &temp); // Copy all values to temporary table - for (auto i (store::iterator (std::make_unique> (transaction_a, pending_store.pending_handle))), n (store::iterator (nullptr)); i != n; ++i) + for (auto i (store::iterator (std::make_unique> (transaction_a, env, pending_store.pending_handle))), n (store::iterator (nullptr)); i != n; ++i) { auto s = mdb_put (env.tx (transaction_a), temp, nano::store::lmdb::db_val (i->first), nano::store::lmdb::db_val (i->second), MDB_APPEND); release_assert_success (s); @@ -403,7 +403,7 @@ void nano::store::lmdb::component::rebuild_db (store::write_transaction const & release_assert (count (transaction_a, pending_store.pending_handle) == count (transaction_a, temp)); mdb_drop (env.tx (transaction_a), pending_store.pending_handle, 0); // Put values from copy - for (auto i (store::iterator (std::make_unique> (transaction_a, temp))), n (store::iterator (nullptr)); i != n; ++i) + for (auto i (store::iterator (std::make_unique> (transaction_a, env, temp))), n (store::iterator (nullptr)); i != n; ++i) { auto s = mdb_put (env.tx (transaction_a), pending_store.pending_handle, nano::store::lmdb::db_val (i->first), nano::store::lmdb::db_val (i->second), MDB_APPEND); release_assert_success (s); diff --git a/nano/store/lmdb/lmdb.hpp b/nano/store/lmdb/lmdb.hpp index cd741d5c9e..738a241266 100644 --- a/nano/store/lmdb/lmdb.hpp +++ b/nano/store/lmdb/lmdb.hpp @@ -104,13 +104,13 @@ class component : public nano::store::component template store::iterator make_iterator (store::transaction const & transaction_a, tables table_a, bool const direction_asc = true) const { - return store::iterator (std::make_unique> (transaction_a, table_to_dbi (table_a), nano::store::lmdb::db_val{}, direction_asc)); + return store::iterator (std::make_unique> (transaction_a, env, table_to_dbi (table_a), nano::store::lmdb::db_val{}, direction_asc)); } template store::iterator make_iterator (store::transaction const & transaction_a, tables table_a, nano::store::lmdb::db_val const & key) const { - return store::iterator (std::make_unique> (transaction_a, table_to_dbi (table_a), key)); + return store::iterator (std::make_unique> (transaction_a, env, table_to_dbi (table_a), key)); } bool init_error () const override; diff --git a/nano/store/lmdb/lmdb_env.cpp b/nano/store/lmdb/lmdb_env.cpp index b34289e7f1..efb5e83087 100644 --- a/nano/store/lmdb/lmdb_env.cpp +++ b/nano/store/lmdb/lmdb_env.cpp @@ -1,3 +1,4 @@ +#include #include #include @@ -105,5 +106,6 @@ nano::store::write_transaction nano::store::lmdb::env::tx_begin_write (store::lm MDB_txn * nano::store::lmdb::env::tx (store::transaction const & transaction_a) const { + debug_assert (transaction_a.store_id () == store_id); return static_cast (transaction_a.get_handle ()); } diff --git a/nano/store/lmdb/lmdb_env.hpp b/nano/store/lmdb/lmdb_env.hpp index 21634509db..1bde4c211c 100644 --- a/nano/store/lmdb/lmdb_env.hpp +++ b/nano/store/lmdb/lmdb_env.hpp @@ -1,9 +1,15 @@ #pragma once +#include #include #include #include +namespace +{ +nano::id_dispenser id_gen; +} + namespace nano::store::lmdb { /** @@ -62,5 +68,6 @@ class env final store::write_transaction tx_begin_write (txn_callbacks callbacks = txn_callbacks{}) const; MDB_txn * tx (store::transaction const & transaction_a) const; MDB_env * environment; + nano::id_dispenser::id_t const store_id{ id_gen.next_id () }; }; } // namespace nano::store::lmdb diff --git a/nano/store/lmdb/transaction.cpp b/nano/store/lmdb/transaction.cpp index ad18863934..a01698c09d 100644 --- a/nano/store/lmdb/transaction.cpp +++ b/nano/store/lmdb/transaction.cpp @@ -36,6 +36,7 @@ class matches_txn final } nano::store::lmdb::read_transaction_impl::read_transaction_impl (nano::store::lmdb::env const & environment_a, nano::store::lmdb::txn_callbacks txn_callbacks_a) : + store::read_transaction_impl (environment_a.store_id), txn_callbacks (txn_callbacks_a) { auto status (mdb_txn_begin (environment_a, nullptr, MDB_RDONLY, &handle)); @@ -70,6 +71,7 @@ void * nano::store::lmdb::read_transaction_impl::get_handle () const } nano::store::lmdb::write_transaction_impl::write_transaction_impl (nano::store::lmdb::env const & environment_a, nano::store::lmdb::txn_callbacks txn_callbacks_a) : + store::write_transaction_impl (environment_a.store_id), env (environment_a), txn_callbacks (txn_callbacks_a) { diff --git a/nano/store/lmdb/transaction_impl.hpp b/nano/store/lmdb/transaction_impl.hpp index cdeff1404e..001fd896a3 100644 --- a/nano/store/lmdb/transaction_impl.hpp +++ b/nano/store/lmdb/transaction_impl.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -27,6 +28,7 @@ class txn_callbacks std::function txn_start{ [] (store::transaction_impl const *) {} }; std::function txn_end{ [] (store::transaction_impl const *) {} }; }; + class read_transaction_impl final : public store::read_transaction_impl { public: @@ -53,7 +55,7 @@ class write_transaction_impl final : public store::write_transaction_impl lmdb::txn_callbacks txn_callbacks; bool active{ true }; }; -} // namespace nano +} // namespace nano::store::lmdb namespace nano { diff --git a/nano/store/transaction.cpp b/nano/store/transaction.cpp index 7d30d403a0..395f81caf9 100644 --- a/nano/store/transaction.cpp +++ b/nano/store/transaction.cpp @@ -2,6 +2,37 @@ #include #include +/* + * transaction_impl + */ + +nano::store::transaction_impl::transaction_impl (nano::id_dispenser::id_t const store_id_a) : + store_id{ store_id_a } +{ +} + +/* + * read_transaction_impl + */ + +nano::store::read_transaction_impl::read_transaction_impl (nano::id_dispenser::id_t const store_id_a) : + transaction_impl (store_id_a) +{ +} + +/* + * write_transaction_impl + */ + +nano::store::write_transaction_impl::write_transaction_impl (nano::id_dispenser::id_t const store_id_a) : + transaction_impl (store_id_a) +{ +} + +/* + * read_transaction + */ + nano::store::read_transaction::read_transaction (std::unique_ptr read_transaction_impl) : impl (std::move (read_transaction_impl)) { @@ -12,6 +43,11 @@ void * nano::store::read_transaction::get_handle () const return impl->get_handle (); } +nano::id_dispenser::id_t nano::store::read_transaction::store_id () const +{ + return impl->store_id; +} + void nano::store::read_transaction::reset () const { impl->reset (); @@ -28,6 +64,10 @@ void nano::store::read_transaction::refresh () const renew (); } +/* + * write_transaction + */ + nano::store::write_transaction::write_transaction (std::unique_ptr write_transaction_impl) : impl (std::move (write_transaction_impl)) { @@ -42,6 +82,11 @@ void * nano::store::write_transaction::get_handle () const return impl->get_handle (); } +nano::id_dispenser::id_t nano::store::write_transaction::store_id () const +{ + return impl->store_id; +} + void nano::store::write_transaction::commit () { impl->commit (); diff --git a/nano/store/transaction.hpp b/nano/store/transaction.hpp index 5061886907..70b2d9c344 100644 --- a/nano/store/transaction.hpp +++ b/nano/store/transaction.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -9,13 +10,17 @@ namespace nano::store class transaction_impl { public: + transaction_impl (nano::id_dispenser::id_t const store_id); virtual ~transaction_impl () = default; virtual void * get_handle () const = 0; + + nano::id_dispenser::id_t const store_id; }; class read_transaction_impl : public transaction_impl { public: + explicit read_transaction_impl (nano::id_dispenser::id_t const store_id = 0); virtual void reset () = 0; virtual void renew () = 0; }; @@ -23,6 +28,7 @@ class read_transaction_impl : public transaction_impl class write_transaction_impl : public transaction_impl { public: + explicit write_transaction_impl (nano::id_dispenser::id_t const store_id = 0); virtual void commit () = 0; virtual void renew () = 0; virtual bool contains (nano::tables table_a) const = 0; @@ -33,6 +39,7 @@ class transaction public: virtual ~transaction () = default; virtual void * get_handle () const = 0; + virtual nano::id_dispenser::id_t store_id () const = 0; }; /** @@ -44,6 +51,8 @@ class read_transaction final : public transaction public: explicit read_transaction (std::unique_ptr read_transaction_impl); void * get_handle () const override; + nano::id_dispenser::id_t store_id () const override; + void reset () const; void renew () const; void refresh () const; @@ -61,6 +70,8 @@ class write_transaction final : public transaction public: explicit write_transaction (std::unique_ptr write_transaction_impl); void * get_handle () const override; + nano::id_dispenser::id_t store_id () const override; + void commit (); void renew (); void refresh ();