Feature/verifytrie admin rpc (#7962)

src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs

@@ -23,6 +23,7 @@
 using Nethermind.JsonRpc;
 using Nethermind.JsonRpc.Modules.Eth.GasPrice;
 using Nethermind.State;
+using Nethermind.Synchronization.FastSync;
 using Nethermind.Trie;
 using Nethermind.Trie.Pruning;
 using Nethermind.TxPool;
@@ -63,6 +64,7 @@ public interface IApiWithBlockchain : IApiWithStores, IBlockchainBridgeFactory
         /// DO NOT USE OUTSIDE OF PROCESSING BLOCK CONTEXT!
         /// </remarks>
         IWorldState? WorldState { get; set; }
+        IBlockingVerifyTrie? BlockingVerifyTrie { get; set; }
         IReadOnlyStateProvider? ChainHeadStateProvider { get; set; }
         IStateReader? StateReader { get; set; }
         IWorldStateManager? WorldStateManager { get; set; }

src/Nethermind/Nethermind.Api/NethermindApi.cs

@@ -58,6 +58,7 @@
 using Nethermind.Trie;
 using Nethermind.Consensus.Processing.CensorshipDetector;
 using Nethermind.Facade.Find;
+using Nethermind.Synchronization.FastSync;
 
 namespace Nethermind.Api
 {
@@ -196,6 +197,7 @@ public ISealEngine SealEngine
         public IPeerDifficultyRefreshPool? PeerDifficultyRefreshPool => ApiWithNetworkServiceContainer?.Resolve<IPeerDifficultyRefreshPool>();
         public ISynchronizer? Synchronizer => ApiWithNetworkServiceContainer?.Resolve<ISynchronizer>();
         public ISyncServer? SyncServer => ApiWithNetworkServiceContainer?.Resolve<ISyncServer>();
+        public IBlockingVerifyTrie? BlockingVerifyTrie { get; set; }
         public IWorldState? WorldState { get; set; }
         public IReadOnlyStateProvider? ChainHeadStateProvider { get; set; }
         public IWorldStateManager? WorldStateManager { get; set; }

src/Nethermind/Nethermind.Init/InitializeStateDb.cs

@@ -22,6 +22,7 @@
 using Nethermind.Logging;
 using Nethermind.Serialization.Json;
 using Nethermind.State;
+using Nethermind.Synchronization.FastSync;
 using Nethermind.Synchronization.Trie;
 using Nethermind.Trie;
 using Nethermind.Trie.Pruning;
@@ -87,7 +88,7 @@ public Task Execute(CancellationToken cancellationToken)
             syncConfig.DownloadBodiesInFastSync = true;
         }
 
-        IKeyValueStore codeDb = getApi.DbProvider.CodeDb;
+        IDb codeDb = getApi.DbProvider.CodeDb;
         IKeyValueStoreWithBatching stateDb = getApi.DbProvider.StateDb;
         IPersistenceStrategy persistenceStrategy;
         IPruningStrategy pruningStrategy;
@@ -185,6 +186,8 @@ public Task Execute(CancellationToken cancellationToken)
             getApi.DbProvider,
             getApi.LogManager);
 
+        setApi.BlockingVerifyTrie = new BlockingVerifyTrie(mainWorldTrieStore, stateManager.GlobalStateReader, codeDb!, getApi.ProcessExit!, getApi.LogManager);
+
         // TODO: Don't forget this
         TrieStoreBoundaryWatcher trieStoreBoundaryWatcher = new(stateManager, _api.BlockTree!, _api.LogManager);
         getApi.DisposeStack.Push(trieStoreBoundaryWatcher);
@@ -199,9 +202,12 @@ public Task Execute(CancellationToken cancellationToken)
         if (_api.Config<IInitConfig>().DiagnosticMode == DiagnosticMode.VerifyTrie)
         {
             _logger!.Info("Collecting trie stats and verifying that no nodes are missing...");
-            Hash256 stateRoot = getApi.BlockTree!.Head?.StateRoot ?? Keccak.EmptyTreeHash;
-            TrieStats stats = stateManager.GlobalStateReader.CollectStats(stateRoot, getApi.DbProvider.CodeDb, _api.LogManager, _api.ProcessExit!.Token);
-            _logger.Info($"Starting from {getApi.BlockTree.Head?.Number} {getApi.BlockTree.Head?.StateRoot}{Environment.NewLine}" + stats);
+            BlockHeader? head = getApi.BlockTree!.Head?.Header;
+            if (head is not null)
+            {
+                setApi.BlockingVerifyTrie.TryStartVerifyTrie(head);
+                _logger.Info($"Starting from {head.Number} {head.StateRoot}{Environment.NewLine}");
+            }
         }
 
         // Init state if we need system calls before actual processing starts

src/Nethermind/Nethermind.Init/Steps/RegisterRpcModules.cs

@@ -127,11 +127,14 @@ public virtual async Task Execute(CancellationToken cancellationToken)
         ManualPruningTrigger pruningTrigger = new();
         _api.PruningTrigger.Add(pruningTrigger);
         (IApiWithStores getFromApi, IApiWithBlockchain setInApi) = _api.ForInit;
+
         AdminRpcModule adminRpcModule = new(
             _api.BlockTree,
             networkConfig,
             _api.PeerPool,
             _api.StaticNodesManager,
+            _api.BlockingVerifyTrie!,
+            _api.WorldStateManager.GlobalStateReader,
             _api.Enode,
             initConfig.BaseDbPath,
             pruningTrigger,

src/Nethermind/Nethermind.JsonRpc.Test/Modules/AdminModuleTests.cs

@@ -18,7 +18,9 @@
 using Nethermind.Network.Config;
 using Nethermind.Serialization.Json;
 using Nethermind.Specs.ChainSpecStyle;
+using Nethermind.State;
 using Nethermind.Stats.Model;
+using Nethermind.Synchronization.FastSync;
 using NSubstitute;
 using NUnit.Framework;
 
@@ -32,13 +34,17 @@ public class AdminModuleTests
     private EthereumJsonSerializer _serializer = null!;
     private NetworkConfig _networkConfig = null!;
     private IBlockTree _blockTree = null!;
+    private IBlockingVerifyTrie _blockingVerifyTrie = null!;
+    private IStateReader _stateReader = null!;
     private const string _enodeString = "enode://e1b7e0dc09aae610c9dec8a0bee62bab9946cc27ebdd2f9e3571ed6d444628f99e91e43f4a14d42d498217608bb3e1d1bc8ec2aa27d7f7e423413b851bae02bc@127.0.0.1:30303";
     private const string _exampleDataDir = "/example/dbdir";
 
     [SetUp]
     public void Setup()
     {
         _blockTree = Build.A.BlockTree().OfChainLength(5).TestObject;
+        _blockingVerifyTrie = Substitute.For<IBlockingVerifyTrie>();
+        _stateReader = Substitute.For<IStateReader>();
         _networkConfig = new NetworkConfig();
         IPeerPool peerPool = Substitute.For<IPeerPool>();
         ConcurrentDictionary<PublicKeyAsKey, Peer> dict = new();
@@ -57,6 +63,8 @@ public void Setup()
             _networkConfig,
             peerPool,
             staticNodesManager,
+            _blockingVerifyTrie,
+            _stateReader,
             enode,
             _exampleDataDir,
             new ManualPruningTrigger(),
@@ -110,6 +118,23 @@ public async Task Test_admin_dataDir()
         response.Result!.ToString().Should().Be(_exampleDataDir);
     }
 
+    [Test]
+    public async Task Test_admin_verifyTrie()
+    {
+        (await RpcTest.TestSerializedRequest(_adminRpcModule, "admin_verifyTrie", "latest")).Should().Contain("Unable to start verify trie");
+        _stateReader.HasStateForRoot(Arg.Any<Hash256>()).Returns(true);
+        _blockingVerifyTrie.TryStartVerifyTrie(Arg.Any<BlockHeader>()).Returns(true);
+        (await RpcTest.TestSerializedRequest(_adminRpcModule, "admin_verifyTrie", "latest")).Should().Contain("Starting");
+    }
+
+    [Test]
+    public async Task Test_hasStateForBlock()
+    {
+        (await RpcTest.TestSerializedRequest(_adminRpcModule, "admin_isStateRootAvailable", "latest")).Should().Contain("false");
+        _stateReader.HasStateForRoot(Arg.Any<Hash256>()).Returns(true);
+        (await RpcTest.TestSerializedRequest(_adminRpcModule, "admin_isStateRootAvailable", "latest")).Should().Contain("true");
+    }
+
     [Test]
     public async Task Smoke_solc()
     {

src/Nethermind/Nethermind.JsonRpc/Modules/Admin/AdminRpcModule.cs

@@ -5,14 +5,17 @@
 using System.Linq;
 using System.Threading.Tasks;
 using Nethermind.Blockchain;
+using Nethermind.Blockchain.Find;
 using Nethermind.Blockchain.FullPruning;
 using Nethermind.Config;
 using Nethermind.Core;
 using Nethermind.Core.Crypto;
 using Nethermind.Network;
 using Nethermind.Network.Config;
 using Nethermind.Specs.ChainSpecStyle;
+using Nethermind.State;
 using Nethermind.Stats.Model;
+using Nethermind.Synchronization.FastSync;
 
 namespace Nethermind.JsonRpc.Modules.Admin;
 
@@ -26,13 +29,17 @@ public class AdminRpcModule : IAdminRpcModule
     private readonly IEnode _enode;
     private readonly string _dataDir;
     private readonly ManualPruningTrigger _pruningTrigger;
+    private readonly IBlockingVerifyTrie _blockingVerifyTrie;
+    private readonly IStateReader _stateReader;
     private NodeInfo _nodeInfo = null!;
 
     public AdminRpcModule(
         IBlockTree blockTree,
         INetworkConfig networkConfig,
         IPeerPool peerPool,
         IStaticNodesManager staticNodesManager,
+        IBlockingVerifyTrie blockingVerifyTrie,
+        IStateReader stateReader,
         IEnode enode,
         string dataDir,
         ManualPruningTrigger pruningTrigger,
@@ -44,6 +51,8 @@ public AdminRpcModule(
         _peerPool = peerPool ?? throw new ArgumentNullException(nameof(peerPool));
         _networkConfig = networkConfig ?? throw new ArgumentNullException(nameof(networkConfig));
         _staticNodesManager = staticNodesManager ?? throw new ArgumentNullException(nameof(staticNodesManager));
+        _blockingVerifyTrie = blockingVerifyTrie ?? throw new ArgumentNullException(nameof(blockingVerifyTrie));
+        _stateReader = stateReader ?? throw new ArgumentNullException(nameof(stateReader));
         _pruningTrigger = pruningTrigger;
         _parameters = parameters ?? throw new ArgumentNullException(nameof(parameters));
 
@@ -135,8 +144,40 @@ public ResultWrapper<bool> admin_setSolc()
         return ResultWrapper<bool>.Success(true);
     }
 
+    public ResultWrapper<bool> admin_isStateRootAvailable(BlockParameter block)
+    {
+        BlockHeader? header = _blockTree.FindHeader(block);
+        if (header is null)
+        {
+            return ResultWrapper<bool>.Fail("Unable to find block. Unable to know state root to verify.");
+        }
+
+        return ResultWrapper<bool>.Success(_stateReader.HasStateForBlock(header));
+    }
+
     public ResultWrapper<PruningStatus> admin_prune()
     {
         return ResultWrapper<PruningStatus>.Success(_pruningTrigger.Trigger());
     }
+
+    public ResultWrapper<string> admin_verifyTrie(BlockParameter block)
+    {
+        BlockHeader? header = _blockTree.FindHeader(block);
+        if (header is null)
+        {
+            return ResultWrapper<string>.Fail("Unable to find block. Unable to know state root to verify.");
+        }
+
+        if (!_stateReader.HasStateForBlock(header))
+        {
+            return ResultWrapper<string>.Fail("Unable to start verify trie. State for block missing.");
+        }
+
+        if (!_blockingVerifyTrie.TryStartVerifyTrie(header))
+        {
+            return ResultWrapper<string>.Fail("Unable to start verify trie. Verify trie already running.");
+        }
+
+        return ResultWrapper<string>.Success("Starting.");
+    }
 }

src/Nethermind/Nethermind.JsonRpc/Modules/Admin/IAdminRpcModule.cs

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: LGPL-3.0-only
 
 using System.Threading.Tasks;
+using Nethermind.Blockchain.Find;
 using Nethermind.Blockchain.FullPruning;
 
 namespace Nethermind.JsonRpc.Modules.Admin;
@@ -57,14 +58,19 @@ ResultWrapper<PeerInfo[]> admin_peers(
         IsImplemented = true)]
     ResultWrapper<string> admin_dataDir();
 
-
     [JsonRpcMethod(Description = "[DEPRECATED]",
         IsImplemented = false)]
     ResultWrapper<bool> admin_setSolc();
 
-    [JsonRpcMethod(Description = "Runs full pruning if enabled.",
+    [JsonRpcMethod(Description = "True if state root for the block is available",
+        EdgeCaseHint = "",
+        ExampleResponse = "\"Starting\"",
+        IsImplemented = true)]
+    ResultWrapper<bool> admin_isStateRootAvailable(BlockParameter block);
+
+    [JsonRpcMethod(Description = "Runs VerifyTrie.",
         EdgeCaseHint = "",
         ExampleResponse = "\"Starting\"",
         IsImplemented = true)]
-    ResultWrapper<PruningStatus> admin_prune();
+    ResultWrapper<string> admin_verifyTrie(BlockParameter block);
 }

src/Nethermind/Nethermind.Runner.Test/Ethereum/ContextWithMocks.cs

@@ -47,6 +47,7 @@
 using Nethermind.Blockchain.Blocks;
 using Nethermind.Core;
 using Nethermind.Facade.Find;
+using Nethermind.Synchronization.FastSync;
 
 namespace Nethermind.Runner.Test.Ethereum
 {
@@ -94,6 +95,7 @@ public static NethermindApi ContextWithMocks()
                 SealValidator = Substitute.For<ISealValidator>(),
                 SessionMonitor = Substitute.For<ISessionMonitor>(),
                 WorldState = Substitute.For<IWorldState>(),
+                BlockingVerifyTrie = Substitute.For<IBlockingVerifyTrie>(),
                 StateReader = Substitute.For<IStateReader>(),
                 TransactionProcessor = Substitute.For<ITransactionProcessor>(),
                 TxSender = Substitute.For<ITxSender>(),

src/Nethermind/Nethermind.Synchronization.Test/SynchronizerModuleTests.cs

@@ -38,63 +38,27 @@ public IContainer CreateTestContainer()
             .AddSingleton(stateReader)
             .AddSingleton(treeSync)
             .AddSingleton(blockQueue)
+            .AddSingleton(Substitute.For<IBlockingVerifyTrie>())
             .AddSingleton(Substitute.For<IProcessExitSource>())
             .AddSingleton<ILogManager>(LimboLogs.Instance)
             .Build();
     }
 
     [Test]
-    public void TestOnTreeSyncFinish_CallVisit()
+    public async Task TestOnTreeSyncFinish_CallVisit()
     {
         IContainer ctx = CreateTestContainer();
         ITreeSync treeSync = ctx.Resolve<ITreeSync>();
-        IStateReader stateReader = ctx.Resolve<IStateReader>();
+        IBlockingVerifyTrie verifyTrie = ctx.Resolve<IBlockingVerifyTrie>();
 
-        treeSync.SyncCompleted += Raise.EventWith(null, new ITreeSync.SyncCompletedEventArgs(TestItem.KeccakA));
-        treeSync.SyncCompleted += Raise.EventWith(null, new ITreeSync.SyncCompletedEventArgs(TestItem.KeccakA));
-
-        stateReader
-            .Received(1)
-            .RunTreeVisitor(Arg.Any<TrieStatsCollector>(), Arg.Is(TestItem.KeccakA), Arg.Any<VisitingOptions>());
-    }
-
-    [Test]
-    public async Task TestOnTreeSyncFinish_BlockProcessingQueue_UntilFinished()
-    {
-        IContainer ctx = CreateTestContainer();
-        ITreeSync treeSync = ctx.Resolve<ITreeSync>();
-        IStateReader stateReader = ctx.Resolve<IStateReader>();
-        IBlockProcessingQueue blockQueue = ctx.Resolve<IBlockProcessingQueue>();
-
-        ManualResetEvent treeVisitorBlocker = new ManualResetEvent(false);
-
-        stateReader
-            .When(sr => sr.RunTreeVisitor(Arg.Any<TrieStatsCollector>(), Arg.Is(TestItem.KeccakA), Arg.Any<VisitingOptions>()))
-            .Do((ci) =>
-            {
-                treeVisitorBlocker.WaitOne();
-            });
-
-        Task triggerTask = Task.Run(() =>
-        {
-            treeSync.SyncCompleted += Raise.EventWith(null, new ITreeSync.SyncCompletedEventArgs(TestItem.KeccakA));
-        });
-
-        await Task.Delay(100);
-
-        Task blockQueueTask = Task.Run(() =>
-        {
-            blockQueue.BlockRemoved +=
-                Raise.EventWith(null, new BlockRemovedEventArgs(null!, ProcessingResult.Success));
-        });
+        BlockHeader header = Build.A.BlockHeader.WithStateRoot(TestItem.KeccakA).TestObject;
+        treeSync.SyncCompleted += Raise.EventWith(null, new ITreeSync.SyncCompletedEventArgs(header));
+        treeSync.SyncCompleted += Raise.EventWith(null, new ITreeSync.SyncCompletedEventArgs(header));
 
         await Task.Delay(100);
 
-        blockQueueTask.IsCompleted.Should().BeFalse();
-        treeVisitorBlocker.Set();
-
-        await triggerTask;
-        await blockQueueTask;
-        blockQueue.BlockRemoved += Raise.EventWith(null, new BlockRemovedEventArgs(null!, ProcessingResult.Success));
+        verifyTrie
+            .Received(1)
+            .TryStartVerifyTrie(Arg.Any<BlockHeader>());
     }
 }