snapshot support state and block seperate

bxq2011hust · Jul 25, 2024 · 0e705aa · 0e705aa
1 parent 59199c8
commit 0e705aa
Show file tree

Hide file tree

Showing 8 changed files with 412 additions and 148 deletions.
diff --git a/bcos-framework/bcos-framework/storage/Common.h b/bcos-framework/bcos-framework/storage/Common.h
@@ -28,13 +28,18 @@
 #include <map>
 #include <memory>
 #include <string>
+#include <string_view>
 #include <utility>
 #include <vector>
 
 #define STORAGE_LOG(LEVEL) BCOS_LOG(LEVEL) << "[STORAGE]"
 
 namespace bcos::storage
 {
+
+const std::string ROCKSDB = "rocksDB";
+const std::string TiKV = "TiKV";
+
 enum StorageError
 {
     UnknownError = -60000,

diff --git a/bcos-ledger/src/libledger/Ledger.cpp b/bcos-ledger/src/libledger/Ledger.cpp
@@ -1359,7 +1359,6 @@ void Ledger::asyncGetBlockTransactionHashes(bcos::protocol::BlockNumber blockNum
 void Ledger::asyncBatchGetTransactions(std::shared_ptr<std::vector<std::string>> hashes,
     std::function<void(Error::Ptr&&, std::vector<protocol::Transaction::Ptr>&&)> callback)
 {
-    // FIXME: adapt the getBlockStorage()
     std::vector<std::string_view> hashesView;
     hashesView.reserve(hashes->size());
     for (auto& hash : *hashes)

diff --git a/bcos-tool/bcos-tool/NodeConfig.cpp b/bcos-tool/bcos-tool/NodeConfig.cpp
@@ -715,6 +715,13 @@ void NodeConfig::loadStorageConfig(boost::property_tree::ptree const& _pt)
     m_pdKeyPath = _pt.get<std::string>("storage.pd_ssl_key_path", "");
     m_enableArchive = _pt.get<bool>("storage.enable_archive", false);
     m_enableSeparateBlockAndState = _pt.get<bool>("storage.enable_separate_block_state", false);
+    if (boost::iequals(m_storageType, bcos::storage::TiKV))
+    {
+        m_enableSeparateBlockAndState = false;
+        NodeConfig_LOG(INFO) << LOG_DESC("Only rocksDB support separate block and state")
+                             << LOG_KV("separateBlockAndState", m_enableSeparateBlockAndState)
+                             << LOG_KV("storageType", m_storageType);
+    }
     m_stateDBPath = m_storagePath;
     m_stateDBPath = m_storagePath + "/state";
     m_blockDBPath = m_storagePath + "/block";

diff --git a/libinitializer/Initializer.cpp b/libinitializer/Initializer.cpp
diff --git a/libinitializer/Initializer.h b/libinitializer/Initializer.h
@@ -33,6 +33,11 @@
 #include "LightNodeInitializer.h"
 #endif
 
+namespace rocksdb
+{
+class Slice;
+}
+
 namespace bcos
 {
 namespace gateway
@@ -86,8 +91,6 @@ class Initializer
     void initSysContract();
     bcos::storage::TransactionalStorageInterface::Ptr storage() { return m_storage; }
     bcos::Error::Ptr generateSnapshot(const std::string& snapshotPath, bool withTxAndReceipts);
-    bcos::Error::Ptr generateSnapshotFromRocksDB(const std::string& rockDBPath,
-        const std::string& snapshotPath, bool withTxAndReceipts, size_t snapshotFileSize = 256);
     bcos::Error::Ptr importSnapshot(const std::string& snapshotPath);
     bcos::Error::Ptr importSnapshotToRocksDB(
         const std::string& snapshotPath, const std::string& rockDBPath);
@@ -119,5 +122,9 @@ class Initializer
 
     protocol::BlockNumber getCurrentBlockNumber();
 };
+
+bcos::Error::Ptr traverseRocksDB(const std::string& rockDBPath,
+    const std::function<bcos::Error::Ptr(const rocksdb::Slice& key, const rocksdb::Slice& value)>&
+        processor);
 }  // namespace initializer
 }  // namespace bcos
diff --git a/tools/archive-tool/ArchiveService.h b/tools/archive-tool/ArchiveService.h
@@ -211,7 +211,7 @@ class ArchiveService : public std::enable_shared_from_this<ArchiveService>
     }
 
     Error::Ptr deleteArchivedData(int64_t startBlock, int64_t endBlock)
-    {
+    {  // delete blocks in [startBlock, endBlock)
         auto blockFlag = bcos::ledger::HEADER;
         for (int64_t blockNumber = startBlock; blockNumber < endBlock; blockNumber++)
         {

diff --git a/tools/archive-tool/archiveTool.cpp b/tools/archive-tool/archiveTool.cpp
@@ -180,6 +180,7 @@ createBackendStorage(std::shared_ptr<bcos::tool::NodeConfig> nodeConfig, const s
                     nodeConfig->blockDBPath(), option, nodeConfig->enableStatistics());
                 blockStorage = StorageInitializer::build(std::move(blockDB), dataEncryption);
             }
+            blockStorage = storage;
         }
         else
         {
@@ -193,6 +194,7 @@ createBackendStorage(std::shared_ptr<bcos::tool::NodeConfig> nodeConfig, const s
                 blockStorage = std::make_shared<RocksDBStorage>(
                     std::unique_ptr<rocksdb::DB>(blockRocksDB), dataEncryption);
             }
+            blockStorage = storage;
         }
     }
     else if (boost::iequals(nodeConfig->storageType(), "TiKV"))
@@ -531,7 +533,7 @@ void reimportBlocks(auto archiveStorage, TransactionalStorageInterface::Ptr loca
                         h256s topics;
                         for (const auto& k : logEntryJson["topics"])
                         {
-                            topics.push_back(h256(k.asString()));
+                            topics.emplace_back(k.asString());
                         }
                         auto address = logEntryJson["address"].asString();
                         auto addr = bytes(address.data(), address.data() + address.size());

diff --git a/vcpkg.json b/vcpkg.json
@@ -47,7 +47,11 @@
     "wedprcrypto",
     "range-v3",
     "hsm-crypto",
-    "tarscpp"
+    "tarscpp",
+    {
+      "name": "tomlplusplus",
+      "version>=": "3.3.0"
+    }
   ],
   "features": {
     "fullnode": {