Skip to content

Commit

Permalink
Merge pull request #2476 from moreal/remote-kv-service
Browse files Browse the repository at this point in the history 
Introduce an option to turn on RemoteKeyValueService optionally
  • Loading branch information
@moreal
moreal authored May 17, 2024
2 parents 9c08c57 + e91cd5a commit 0b6bb13
Show file tree
Hide file tree
Showing 20 changed files with 415 additions and 190 deletions.
4 changes: 4 additions & 0 deletions Libplanet.Headless/Hosting/LibplanetNodeService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ public class LibplanetNodeService : BackgroundService, IDisposable

public readonly IStateStore StateStore;

public readonly IKeyValueStore StateKeyValueStore;

public readonly BlockChain BlockChain;

public readonly Swarm Swarm;
Expand Down Expand Up @@ -174,6 +176,8 @@ IActionEvaluator BuildActionEvaluator(IActionEvaluatorConfiguration actionEvalua
);
}

StateKeyValueStore = keyValueStore;

_obsoletedChainIds = chainIds.Where(chainId => chainId != BlockChain.Id).ToList();

_exceptionHandlerAction = exceptionHandlerAction;
Expand Down
40 changes: 20 additions & 20 deletions NineChronicles.Headless.Executable.sln
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,8 +72,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Libplanet.Crypto", "Lib9c\.
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Libplanet.Extensions.ActionEvaluatorCommonComponents", "Lib9c\.Libplanet.Extensions.ActionEvaluatorCommonComponents\Libplanet.Extensions.ActionEvaluatorCommonComponents.csproj", "{A6922395-36E5-4B0A-BEBD-9BCE34D08722}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Libplanet.Extensions.RemoteBlockChainStates", "Lib9c\.Libplanet.Extensions.RemoteBlockChainStates\Libplanet.Extensions.RemoteBlockChainStates.csproj", "{8F9E5505-C157-4DF3-A419-FF0108731397}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NineChronicles.Headless.AccessControlCenter", "NineChronicles.Headless.AccessControlCenter\NineChronicles.Headless.AccessControlCenter.csproj", "{162C0F4B-A1D9-4132-BC34-31F1247BC26B}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Libplanet.Extensions.PluggedActionEvaluator", "Libplanet.Extensions.PluggedActionEvaluator\Libplanet.Extensions.PluggedActionEvaluator.csproj", "{DE91C36D-3999-47B6-A0BD-848C8EBA2A76}"
Expand All @@ -82,6 +80,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Lib9c.Plugin.Shared", "Lib9
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Libplanet.Mocks", "Lib9c\.Libplanet\Libplanet.Mocks\Libplanet.Mocks.csproj", "{F79B695B-6FCC-43F5-AEE4-88E484382B9B}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Libplanet.Store.Remote", "Lib9c\.Libplanet\Libplanet.Store.Remote\Libplanet.Store.Remote.csproj", "{D1E15F81-8765-4DCA-9299-675415686C23}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -653,24 +653,6 @@ Global
{A6922395-36E5-4B0A-BEBD-9BCE34D08722}.Release|x64.Build.0 = Release|Any CPU
{A6922395-36E5-4B0A-BEBD-9BCE34D08722}.Release|x86.ActiveCfg = Release|Any CPU
{A6922395-36E5-4B0A-BEBD-9BCE34D08722}.Release|x86.Build.0 = Release|Any CPU
{8F9E5505-C157-4DF3-A419-FF0108731397}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{8F9E5505-C157-4DF3-A419-FF0108731397}.Debug|Any CPU.Build.0 = Debug|Any CPU
{8F9E5505-C157-4DF3-A419-FF0108731397}.Debug|x64.ActiveCfg = Debug|Any CPU
{8F9E5505-C157-4DF3-A419-FF0108731397}.Debug|x64.Build.0 = Debug|Any CPU
{8F9E5505-C157-4DF3-A419-FF0108731397}.Debug|x86.ActiveCfg = Debug|Any CPU
{8F9E5505-C157-4DF3-A419-FF0108731397}.Debug|x86.Build.0 = Debug|Any CPU
{8F9E5505-C157-4DF3-A419-FF0108731397}.DevEx|Any CPU.ActiveCfg = Debug|Any CPU
{8F9E5505-C157-4DF3-A419-FF0108731397}.DevEx|Any CPU.Build.0 = Debug|Any CPU
{8F9E5505-C157-4DF3-A419-FF0108731397}.DevEx|x64.ActiveCfg = Debug|Any CPU
{8F9E5505-C157-4DF3-A419-FF0108731397}.DevEx|x64.Build.0 = Debug|Any CPU
{8F9E5505-C157-4DF3-A419-FF0108731397}.DevEx|x86.ActiveCfg = Debug|Any CPU
{8F9E5505-C157-4DF3-A419-FF0108731397}.DevEx|x86.Build.0 = Debug|Any CPU
{8F9E5505-C157-4DF3-A419-FF0108731397}.Release|Any CPU.ActiveCfg = Release|Any CPU
{8F9E5505-C157-4DF3-A419-FF0108731397}.Release|Any CPU.Build.0 = Release|Any CPU
{8F9E5505-C157-4DF3-A419-FF0108731397}.Release|x64.ActiveCfg = Release|Any CPU
{8F9E5505-C157-4DF3-A419-FF0108731397}.Release|x64.Build.0 = Release|Any CPU
{8F9E5505-C157-4DF3-A419-FF0108731397}.Release|x86.ActiveCfg = Release|Any CPU
{8F9E5505-C157-4DF3-A419-FF0108731397}.Release|x86.Build.0 = Release|Any CPU
{162C0F4B-A1D9-4132-BC34-31F1247BC26B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{162C0F4B-A1D9-4132-BC34-31F1247BC26B}.Debug|Any CPU.Build.0 = Debug|Any CPU
{162C0F4B-A1D9-4132-BC34-31F1247BC26B}.Debug|x64.ActiveCfg = Debug|Any CPU
Expand Down Expand Up @@ -743,6 +725,24 @@ Global
{F79B695B-6FCC-43F5-AEE4-88E484382B9B}.Release|x64.Build.0 = Release|Any CPU
{F79B695B-6FCC-43F5-AEE4-88E484382B9B}.Release|x86.ActiveCfg = Release|Any CPU
{F79B695B-6FCC-43F5-AEE4-88E484382B9B}.Release|x86.Build.0 = Release|Any CPU
{D1E15F81-8765-4DCA-9299-675415686C23}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{D1E15F81-8765-4DCA-9299-675415686C23}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D1E15F81-8765-4DCA-9299-675415686C23}.Debug|x64.ActiveCfg = Debug|Any CPU
{D1E15F81-8765-4DCA-9299-675415686C23}.Debug|x64.Build.0 = Debug|Any CPU
{D1E15F81-8765-4DCA-9299-675415686C23}.Debug|x86.ActiveCfg = Debug|Any CPU
{D1E15F81-8765-4DCA-9299-675415686C23}.Debug|x86.Build.0 = Debug|Any CPU
{D1E15F81-8765-4DCA-9299-675415686C23}.DevEx|Any CPU.ActiveCfg = Debug|Any CPU
{D1E15F81-8765-4DCA-9299-675415686C23}.DevEx|Any CPU.Build.0 = Debug|Any CPU
{D1E15F81-8765-4DCA-9299-675415686C23}.DevEx|x64.ActiveCfg = Debug|Any CPU
{D1E15F81-8765-4DCA-9299-675415686C23}.DevEx|x64.Build.0 = Debug|Any CPU
{D1E15F81-8765-4DCA-9299-675415686C23}.DevEx|x86.ActiveCfg = Debug|Any CPU
{D1E15F81-8765-4DCA-9299-675415686C23}.DevEx|x86.Build.0 = Debug|Any CPU
{D1E15F81-8765-4DCA-9299-675415686C23}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D1E15F81-8765-4DCA-9299-675415686C23}.Release|Any CPU.Build.0 = Release|Any CPU
{D1E15F81-8765-4DCA-9299-675415686C23}.Release|x64.ActiveCfg = Release|Any CPU
{D1E15F81-8765-4DCA-9299-675415686C23}.Release|x64.Build.0 = Release|Any CPU
{D1E15F81-8765-4DCA-9299-675415686C23}.Release|x86.ActiveCfg = Release|Any CPU
{D1E15F81-8765-4DCA-9299-675415686C23}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down
3 changes: 3 additions & 0 deletions NineChronicles.Headless.Executable/Commands/ReplayCommand.Queries.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,9 @@ query GetBlockData($hash: ID!)
preEvaluationHash
previousBlock {
hash
stateRootHash
}
stateRootHash
}
}}
}
Expand Down Expand Up @@ -87,6 +89,7 @@ private sealed class BlockType
public string? Miner { get; set; }
public string? PreEvaluationHash { get; set; }
public BlockType? PreviousBlock { get; set; }
public string? StateRootHash { get; set; }
}

private sealed class TransactionQueryType
Expand Down
42 changes: 34 additions & 8 deletions NineChronicles.Headless.Executable/Commands/ReplayCommand.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,17 +14,19 @@
using Cocona.Help;
using GraphQL.Client.Http;
using GraphQL.Client.Serializer.SystemTextJson;
using Grpc.Net.Client;
using Libplanet.Action;
using Libplanet.Action.Loader;
using Libplanet.Blockchain;
using Libplanet.Blockchain.Policies;
using Libplanet.Extensions.RemoteBlockChainStates;
using Libplanet.Types.Blocks;
using Libplanet.RocksDBStore;
using Libplanet.Action.State;
using Libplanet.Common;
using Libplanet.Crypto;
using Libplanet.Store;
using Libplanet.Store.Remote;
using Libplanet.Store.Remote.Client;
using Libplanet.Types.Tx;
using Nekoyume.Action;
using Nekoyume.Action.Loader;
Expand Down Expand Up @@ -353,8 +355,10 @@ public int Blocks(
public int RemoteTx(
[Option("tx", new[] { 't' }, Description = "The transaction id")]
string transactionId,
[Option("endpoint", new[] { 'e' }, Description = "GraphQL endpoint to get remote state")]
string endpoint)
[Option("endpoint", new[] { 'e' }, Description = "GraphQL endpoint to get block data.")]
string endpoint,
[Option("grpc-endpoint", new []{ 'g' }, Description = "gRPC endpoint to get remote states.")]
string grpcEndpoint)
{
var graphQlClient = new GraphQLHttpClient(new Uri(endpoint), new SystemTextJsonSerializer());
var transactionResponse = GetTransactionData(graphQlClient, transactionId);
Expand All @@ -378,19 +382,41 @@ public int RemoteTx(

var block = GetBlockData(graphQlClient, transactionResult.BlockHash)?.ChainQuery?.BlockQuery?.Block;
var previousBlockHashValue = block?.PreviousBlock?.Hash;
var previousBlockHashStateRootHash = block?.PreviousBlock?.StateRootHash;
var preEvaluationHashValue = block?.PreEvaluationHash;
var minerValue = block?.Miner;
if (previousBlockHashValue is null || preEvaluationHashValue is null || minerValue is null)
if (previousBlockHashValue is null || preEvaluationHashValue is null || minerValue is null || previousBlockHashStateRootHash is null)
{
throw new CommandExitedException("Failed to get block from query", -1);
}
var miner = new Address(minerValue);

var explorerEndpoint = $"{endpoint}/explorer";
var blockChainStates = new RemoteBlockChainStates(new Uri(explorerEndpoint));
var channel = GrpcChannel.ForAddress(grpcEndpoint);
var keyValueServiceClient = new KeyValueStore.KeyValueStoreClient(channel);
var cacheKeyValueStore =
new RocksDBKeyValueStore(Path.Combine(Path.GetTempPath(), "9c-headless-replay-remotetx"));
var keyValueStore = new ReplicableKeyValueStore(
new RemoteKeyValueStore(keyValueServiceClient),
cacheKeyValueStore);
var store = new AnonymousStore
{
GetBlockDigest = hash =>
{
var stateRootHash = HashDigest<SHA256>.FromString(block!.StateRootHash!);
var privateKey = new PrivateKey();
var blockMetadata = new BlockMetadata(
0, DateTimeOffset.Now, privateKey.PublicKey, null, null, null);
var blockContent = new BlockContent(blockMetadata);
var preEvaluationBlock = blockContent.Propose();
var b = preEvaluationBlock.Sign(privateKey, stateRootHash);
return new BlockDigest(b.Header, ImmutableArray<ImmutableArray<byte>>.Empty);
}
};

var previousStateRootHash = HashDigest<SHA256>.FromString(previousBlockHashStateRootHash);
var blockChainStates = new BlockChainStates(store, new TrieStateStore(keyValueStore));

var previousBlockHash = BlockHash.FromString(previousBlockHashValue);
var previousStates = new World(blockChainStates.GetWorldState(previousBlockHash));
var previousStates = new World(blockChainStates.GetWorldState(previousStateRootHash));

var actions = transaction.Actions
.Select(ToAction)
Expand Down
9 changes: 6 additions & 3 deletions NineChronicles.Headless.Executable/Configuration.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,9 @@ public class Configuration
public bool? RpcRemoteServer { get; set; }
public bool? RpcHttpServer { get; set; }

// RemoteKeyValueService
public bool RemoteKeyValueService { get; set; } = false;

// GraphQL Server
public bool GraphQLServer { get; set; }
public string? GraphQLHost { get; set; }
Expand Down Expand Up @@ -92,8 +95,6 @@ public class Configuration

public double SentryTraceSampleRate { get; set; } = 0.01;

public StateServiceManagerServiceOptions? StateServiceManagerService { get; set; }

public AccessControlServiceOptions? AccessControlService { get; set; }

public int ArenaParticipantsSyncInterval { get; set; } = 1000;
Expand Down Expand Up @@ -148,7 +149,8 @@ public void Overwrite(
int? maxTransactionPerBlock,
string? sentryDsn,
double? sentryTraceSampleRate,
int? arenaParticipantsSyncInterval
int? arenaParticipantsSyncInterval,
bool? remoteKeyValueService
)
{
AppProtocolVersionString = appProtocolVersionString ?? AppProtocolVersionString;
Expand Down Expand Up @@ -202,6 +204,7 @@ public void Overwrite(
SentryDsn = sentryDsn ?? SentryDsn;
SentryTraceSampleRate = sentryTraceSampleRate ?? SentryTraceSampleRate;
ArenaParticipantsSyncInterval = arenaParticipantsSyncInterval ?? ArenaParticipantsSyncInterval;
RemoteKeyValueService = remoteKeyValueService ?? RemoteKeyValueService;
}
}
}
3 changes: 2 additions & 1 deletion NineChronicles.Headless.Executable/NineChronicles.Headless.Executable.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
<ItemGroup>
<PackageReference Include="Cocona.Lite" Version="2.1.*" />
<PackageReference Include="Cocona.Docs" Version="0.2.0-dev.2023103141721" />
<PackageReference Include="GraphQL.Client" Version="5.1.1" />
<PackageReference Include="GraphQL.Client.Serializer.SystemTextJson" Version="5.1.1" />
<PackageReference Include="Destructurama.Attributed" Version="2.0.0" />
<PackageReference Include="Sentry.Serilog" Version="3.23.0" />
<PackageReference Include="Sentry" Version="3.23.0" />
Expand All @@ -46,7 +48,6 @@
<ProjectReference Include="..\Lib9c\.Libplanet\Libplanet.Extensions.Cocona\Libplanet.Extensions.Cocona.csproj" />
<ProjectReference Include="..\Lib9c\Lib9c.Utils\Lib9c.Utils.csproj" />
<ProjectReference Include="..\Lib9c\Lib9c.Abstractions\Lib9c.Abstractions.csproj" />
<ProjectReference Include="..\Lib9c\.Libplanet.Extensions.RemoteBlockChainStates\Libplanet.Extensions.RemoteBlockChainStates.csproj" />
<ProjectReference Include="..\NineChronicles.Headless\NineChronicles.Headless.csproj" />
<ProjectReference Include="..\Lib9c\Lib9c.DevExtensions\Lib9c.DevExtensions.csproj" />
</ItemGroup>
Expand Down
43 changes: 5 additions & 38 deletions NineChronicles.Headless.Executable/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,8 @@ public async Task Run(
int? arenaParticipantsSyncInterval = null,
[Option(Description = "arena participants list sync enable")]
bool arenaParticipantsSync = true,
[Option(Description = "[DANGER] Turn on RemoteKeyValueService to debug.")]
bool remoteKeyValueService = false,
[Ignore] CancellationToken? cancellationToken = null
)
{
Expand Down Expand Up @@ -308,7 +310,7 @@ public async Task Run(
txLifeTime, messageTimeout, tipTimeout, demandBuffer, skipPreload,
minimumBroadcastTarget, bucketSize, chainTipStaleBehaviorType, txQuotaPerSigner, maximumPollPeers,
consensusPort, consensusPrivateKeyString, consensusSeedStrings, consensusTargetBlockIntervalMilliseconds, consensusProposeSecondBase,
maxTransactionPerBlock, sentryDsn, sentryTraceSampleRate, arenaParticipantsSyncInterval
maxTransactionPerBlock, sentryDsn, sentryTraceSampleRate, arenaParticipantsSyncInterval, remoteKeyValueService
);

#if SENTRY || ! DEBUG
Expand Down Expand Up @@ -366,11 +368,6 @@ public async Task Run(
);
}

if (headlessConfig.StateServiceManagerService is { } stateServiceManagerServiceOptions)
{
await DownloadStateServices(stateServiceManagerServiceOptions);
}

try
{
IHostBuilder hostBuilder = Host.CreateDefaultBuilder();
Expand Down Expand Up @@ -464,7 +461,7 @@ IActionLoader MakeSingleActionLoader()
: new PrivateKey(ByteUtil.ParseHex(headlessConfig.MinerPrivateKeyString));
TimeSpan minerBlockInterval = TimeSpan.FromMilliseconds(headlessConfig.MinerBlockIntervalMilliseconds);
var nineChroniclesProperties =
new NineChroniclesNodeServiceProperties(actionLoader, headlessConfig.StateServiceManagerService, headlessConfig.AccessControlService)
new NineChroniclesNodeServiceProperties(actionLoader, headlessConfig.AccessControlService)
{
MinerPrivateKey = minerPrivateKey,
Libplanet = properties,
Expand Down Expand Up @@ -579,6 +576,7 @@ IActionLoader MakeSingleActionLoader()
SecretToken = secretToken,
NoCors = headlessConfig.NoCors,
UseMagicOnion = headlessConfig.RpcServer,
UseRemoteKeyValueService = headlessConfig.RemoteKeyValueService,
HttpOptions = headlessConfig.RpcServer && headlessConfig.RpcHttpServer == true
? new GraphQLNodeServiceProperties.MagicOnionHttpOptions(
$"{headlessConfig.RpcListenHost}:{headlessConfig.RpcListenPort}")
Expand Down Expand Up @@ -618,36 +616,5 @@ IActionLoader MakeSingleActionLoader()
static void ConfigureSentryOptions(SentryOptions o)
{
}

private static Task DownloadStateServices(StateServiceManagerServiceOptions options)
{
Log.Information("Downloading StateServices...");

if (Directory.Exists(options.StateServicesDownloadPath))
{
Directory.Delete(options.StateServicesDownloadPath, true);
}

Directory.CreateDirectory(options.StateServicesDownloadPath);

async Task DownloadStateService(string url)
{
var hashed =
Convert.ToHexString(HashDigest<SHA256>.DeriveFrom(Encoding.UTF8.GetBytes(url)).ToByteArray());
var logger = Log.ForContext("StateService", hashed);
using var httpClient = new HttpClient();
var downloadPath = Path.Join(options.StateServicesDownloadPath, hashed + ".zip");
var extractPath = Path.Join(options.StateServicesDownloadPath, hashed);
logger.Debug("Downloading...");
await File.WriteAllBytesAsync(downloadPath, await httpClient.GetByteArrayAsync(url));
logger.Debug("Finished downloading.");
logger.Debug("Extracting...");
ZipFile.ExtractToDirectory(downloadPath, extractPath);
logger.Debug("Finished extracting.");
}

return Task.WhenAll(options.StateServices.Select(stateService => DownloadStateService(stateService.Path)))
.ContinueWith(_ => Log.Information("Finished downloading StateServices..."));
}
}
}
Loading

0 comments on commit 0b6bb13

Please sign in to comment.