From 8a9d08aca25686ecf7a840f51c8af068cc1c529a Mon Sep 17 00:00:00 2001 From: Yang Chun Ung Date: Thu, 12 Dec 2024 21:03:20 +0900 Subject: [PATCH 01/10] Use StateRootHash instead of BlockHash --- NineChronicles.Headless/BlockChainService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NineChronicles.Headless/BlockChainService.cs b/NineChronicles.Headless/BlockChainService.cs index 0c2bff96f..7553cc7b4 100644 --- a/NineChronicles.Headless/BlockChainService.cs +++ b/NineChronicles.Headless/BlockChainService.cs @@ -277,7 +277,7 @@ public UnaryResult> GetSheets( sw.Restart(); if (addresses.Any()) { - var stateRootHash = new BlockHash(stateRootHashBytes); + var stateRootHash = new HashDigest(stateRootHashBytes); IReadOnlyList values = _blockChain.GetWorldState(stateRootHash).GetLegacyStates(addresses); sw.Stop(); Log.Information("[GetSheets]Get sheet from state: {Count}, Elapsed: {Elapsed}", addresses.Count, sw.Elapsed); From 0bd18c8ad632813e617f411dac52976c8d092613 Mon Sep 17 00:00:00 2001 From: eugene-hong <58686228+eugene-doobu@users.noreply.github.com> Date: Fri, 13 Dec 2024 13:36:04 +0900 Subject: [PATCH 02/10] bump lib9c --- Lib9c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib9c b/Lib9c index 487fca896..af4013a61 160000 --- a/Lib9c +++ b/Lib9c @@ -1 +1 @@ -Subproject commit 487fca8967c8f867cf85dd78b17f7e7c84f4efaf +Subproject commit af4013a61648cc178a27ca5d50a89b0f7e21c97a From 948e515912cd4fb29ceabbf28a39882c9c8da422 Mon Sep 17 00:00:00 2001 From: Chanhyuck Ko Date: Fri, 13 Dec 2024 16:38:21 +0900 Subject: [PATCH 03/10] bump: libplanet 5.4.2 --- Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Build.props b/Directory.Build.props index c95b6df16..ec7a28c36 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -1,5 +1,5 @@ - 5.4.1 + 5.4.2 From 2d168ff2921c8c4559a2e9ab38ed29daf3550c98 Mon Sep 17 00:00:00 2001 From: Chanhyuck Ko Date: Sun, 8 Dec 2024 15:34:02 +0900 Subject: [PATCH 04/10] feat: delay precommit option (cherry picked from commit 8baa875b5d954453600dbc54025273d3c50c61d2) (cherry picked from commit d4efac73e6d66a24bb94c8f7296fff5afd5b91de) --- .../Hosting/LibplanetNodeService.cs | 2 +- .../Hosting/LibplanetNodeServiceProperties.cs | 2 +- .../Configuration.cs | 9 ++- NineChronicles.Headless.Executable/Program.cs | 65 +++++++++++++++---- .../NineChroniclesNodeServiceProperties.cs | 12 ++-- 5 files changed, 69 insertions(+), 21 deletions(-) diff --git a/Libplanet.Headless/Hosting/LibplanetNodeService.cs b/Libplanet.Headless/Hosting/LibplanetNodeService.cs index c0b0c3f46..d9b353f56 100644 --- a/Libplanet.Headless/Hosting/LibplanetNodeService.cs +++ b/Libplanet.Headless/Hosting/LibplanetNodeService.cs @@ -238,7 +238,7 @@ IActionEvaluator BuildActionEvaluator(IActionEvaluatorConfiguration actionEvalua ConsensusPrivateKey = Properties.ConsensusPrivateKey, ConsensusWorkers = 500, TargetBlockInterval = TimeSpan.FromMilliseconds(Properties.ConsensusTargetBlockIntervalMilliseconds ?? 7000), - ContextTimeoutOptions = Properties.ContextTimeoutOption, + ContextOption = Properties.ContextOption, }; } diff --git a/Libplanet.Headless/Hosting/LibplanetNodeServiceProperties.cs b/Libplanet.Headless/Hosting/LibplanetNodeServiceProperties.cs index b174f398c..5d93d6a0c 100644 --- a/Libplanet.Headless/Hosting/LibplanetNodeServiceProperties.cs +++ b/Libplanet.Headless/Hosting/LibplanetNodeServiceProperties.cs @@ -68,7 +68,7 @@ public class LibplanetNodeServiceProperties public TimeSpan TipTimeout { get; set; } = TimeSpan.FromSeconds(60); - public ContextTimeoutOption ContextTimeoutOption { get; set; } + public ContextOption ContextOption { get; set; } public int DemandBuffer { get; set; } = 1150; diff --git a/NineChronicles.Headless.Executable/Configuration.cs b/NineChronicles.Headless.Executable/Configuration.cs index 0f0d9c6f2..2606e8363 100644 --- a/NineChronicles.Headless.Executable/Configuration.cs +++ b/NineChronicles.Headless.Executable/Configuration.cs @@ -87,7 +87,8 @@ public class Configuration public string[]? ConsensusSeedStrings { get; set; } public ushort? ConsensusPort { get; set; } public double? ConsensusTargetBlockIntervalMilliseconds { get; set; } - public int? ConsensusProposeSecondBase { get; set; } + public int? ConsensusProposeTimeoutBase { get; set; } + public int? ConsensusEnterPreCommitDelay { get; set; } public int? MaxTransactionPerBlock { get; set; } @@ -139,7 +140,8 @@ public void Overwrite( string? consensusPrivateKeyString, string[]? consensusSeedStrings, double? consensusTargetBlockIntervalMilliseconds, - int? consensusProposeSecondBase, + int? consensusProposeTimeoutBase, + int? consensusEnterPreCommitDelay, int? maxTransactionPerBlock, bool? remoteKeyValueService ) @@ -190,7 +192,8 @@ public void Overwrite( ConsensusSeedStrings = consensusSeedStrings ?? ConsensusSeedStrings; ConsensusPrivateKeyString = consensusPrivateKeyString ?? ConsensusPrivateKeyString; ConsensusTargetBlockIntervalMilliseconds = consensusTargetBlockIntervalMilliseconds ?? ConsensusTargetBlockIntervalMilliseconds; - ConsensusProposeSecondBase = consensusProposeSecondBase ?? ConsensusProposeSecondBase; + ConsensusProposeTimeoutBase = consensusProposeTimeoutBase ?? ConsensusProposeTimeoutBase; + ConsensusEnterPreCommitDelay = consensusEnterPreCommitDelay ?? ConsensusEnterPreCommitDelay; MaxTransactionPerBlock = maxTransactionPerBlock ?? MaxTransactionPerBlock; RemoteKeyValueService = remoteKeyValueService ?? RemoteKeyValueService; } diff --git a/NineChronicles.Headless.Executable/Program.cs b/NineChronicles.Headless.Executable/Program.cs index 8caff2e51..bd500945c 100644 --- a/NineChronicles.Headless.Executable/Program.cs +++ b/NineChronicles.Headless.Executable/Program.cs @@ -209,6 +209,9 @@ public async Task Run( [Option("consensus-propose-second-base", Description = "A propose second base for consensus context timeout. The unit is second.")] int? consensusProposeSecondBase = null, + [Option("consensus-enter-precommit-delay", + Description = "A precommit delay manually set. The unit is millisecond.")] + int? consensusEnterPreCommitDelay = null, [Option("maximum-transaction-per-block", Description = "Maximum transactions allowed in a block. null by default.")] int? maxTransactionPerBlock = null, @@ -290,16 +293,55 @@ public async Task Run( GetActionEvaluatorConfiguration(configuration.GetSection("Headless").GetSection("ActionEvaluator")); headlessConfig.Overwrite( - appProtocolVersionToken, trustedAppProtocolVersionSigners, genesisBlockPath, host, port, - swarmPrivateKeyString, storeType, storePath, noReduceStore, noMiner, minerCount, - minerPrivateKeyString, minerBlockIntervalMilliseconds, planet, iceServerStrings, peerStrings, rpcServer, rpcListenHost, - rpcListenPort, rpcRemoteServer, rpcHttpServer, graphQLServer, graphQLHost, graphQLPort, - graphQLSecretTokenPath, noCors, nonblockRenderer, nonblockRendererQueue, strictRendering, - logActionRenders, confirmations, - txLifeTime, messageTimeout, tipTimeout, demandBuffer, skipPreload, - minimumBroadcastTarget, bucketSize, chainTipStaleBehaviorType, txQuotaPerSigner, maximumPollPeers, - consensusPort, consensusPrivateKeyString, consensusSeedStrings, consensusTargetBlockIntervalMilliseconds, consensusProposeSecondBase, - maxTransactionPerBlock, remoteKeyValueService + appProtocolVersionToken, + trustedAppProtocolVersionSigners, + genesisBlockPath, + host, + port, + swarmPrivateKeyString, + storeType, + storePath, + noReduceStore, + noMiner, + minerCount, + minerPrivateKeyString, + minerBlockIntervalMilliseconds, + planet, + iceServerStrings, + peerStrings, + rpcServer, + rpcListenHost, + rpcListenPort, + rpcRemoteServer, + rpcHttpServer, + graphQLServer, + graphQLHost, + graphQLPort, + graphQLSecretTokenPath, + noCors, + nonblockRenderer, + nonblockRendererQueue, + strictRendering, + logActionRenders, + confirmations, + txLifeTime, + messageTimeout, + tipTimeout, + demandBuffer, + skipPreload, + minimumBroadcastTarget, + bucketSize, + chainTipStaleBehaviorType, + txQuotaPerSigner, + maximumPollPeers, + consensusPort, + consensusPrivateKeyString, + consensusSeedStrings, + consensusTargetBlockIntervalMilliseconds, + consensusProposeSecondBase * 1_000, + consensusEnterPreCommitDelay, + maxTransactionPerBlock, + remoteKeyValueService ); // Clean-up previous temporary log files. @@ -367,7 +409,8 @@ public async Task Run( consensusPrivateKeyString: headlessConfig.ConsensusPrivateKeyString, consensusSeedStrings: headlessConfig.ConsensusSeedStrings, consensusTargetBlockIntervalMilliseconds: headlessConfig.ConsensusTargetBlockIntervalMilliseconds, - consensusProposeSecondBase: headlessConfig.ConsensusProposeSecondBase, + consensusProposeTimeoutBase: headlessConfig.ConsensusProposeTimeoutBase, + consensusEnterPreCommitDelay: headlessConfig.ConsensusEnterPreCommitDelay, maximumPollPeers: headlessConfig.MaximumPollPeers, actionEvaluatorConfiguration: actionEvaluatorConfiguration ); diff --git a/NineChronicles.Headless/Properties/NineChroniclesNodeServiceProperties.cs b/NineChronicles.Headless/Properties/NineChroniclesNodeServiceProperties.cs index f60740a03..edb306290 100644 --- a/NineChronicles.Headless/Properties/NineChroniclesNodeServiceProperties.cs +++ b/NineChronicles.Headless/Properties/NineChroniclesNodeServiceProperties.cs @@ -91,7 +91,8 @@ public static LibplanetNodeServiceProperties string? consensusPrivateKeyString = null, string[]? consensusSeedStrings = null, double? consensusTargetBlockIntervalMilliseconds = null, - int? consensusProposeSecondBase = null, + int? consensusProposeTimeoutBase = null, + int? consensusEnterPreCommitDelay = null, IActionEvaluatorConfiguration? actionEvaluatorConfiguration = null) { var swarmPrivateKey = string.IsNullOrEmpty(swarmPrivateKeyString) @@ -108,9 +109,10 @@ public static LibplanetNodeServiceProperties var peers = peerStrings.Select(PropertyParser.ParsePeer).ToImmutableArray(); var consensusSeeds = consensusSeedStrings?.Select(PropertyParser.ParsePeer).ToImmutableList(); - var consensusContextTimeoutOption = consensusProposeSecondBase.HasValue - ? new ContextTimeoutOption(consensusProposeSecondBase.Value) - : new ContextTimeoutOption(); + var defaultContextOption = new ContextOption(); + var consensusContextOption = new ContextOption( + proposeTimeoutBase: consensusProposeTimeoutBase ?? defaultContextOption.ProposeTimeoutBase, + enterPreCommitDelay: consensusEnterPreCommitDelay ?? defaultContextOption.EnterPreCommitDelay); return new LibplanetNodeServiceProperties { @@ -147,7 +149,7 @@ public static LibplanetNodeServiceProperties ConsensusSeeds = consensusSeeds, ConsensusPrivateKey = consensusPrivateKey, ConsensusTargetBlockIntervalMilliseconds = consensusTargetBlockIntervalMilliseconds, - ContextTimeoutOption = consensusContextTimeoutOption, + ContextOption = consensusContextOption, ActionEvaluatorConfiguration = actionEvaluatorConfiguration ?? new DefaultActionEvaluatorConfiguration(), }; } From 22aca9299cae6eddf23318de2665ee5cb7ea5a24 Mon Sep 17 00:00:00 2001 From: hyeon Date: Mon, 16 Dec 2024 09:47:48 +0900 Subject: [PATCH 05/10] Bump Libplanet to 5.4.2 --- Directory.Build.props | 2 +- Lib9c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index c95b6df16..ec7a28c36 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -1,5 +1,5 @@ - 5.4.1 + 5.4.2 diff --git a/Lib9c b/Lib9c index af4013a61..0e0124be7 160000 --- a/Lib9c +++ b/Lib9c @@ -1 +1 @@ -Subproject commit af4013a61648cc178a27ca5d50a89b0f7e21c97a +Subproject commit 0e0124be7831d85e682b534a64e16ca196c10917 From 23217ce88d653dc179c3cbfdf64dfbbfd97a0d8e Mon Sep 17 00:00:00 2001 From: area363 Date: Mon, 9 Dec 2024 10:12:21 +0900 Subject: [PATCH 06/10] override rate-limiter when using jwt --- NineChronicles.Headless/GraphQLService.cs | 34 +++++---- .../Middleware/CustomRateLimitMiddleware.cs | 72 ++++++++++++++++--- .../Middleware/HttpCaptureMiddleware.cs | 1 + .../HttpMultiAccountManagementMiddleware.cs | 58 ++++++++++++++- 4 files changed, 138 insertions(+), 27 deletions(-) diff --git a/NineChronicles.Headless/GraphQLService.cs b/NineChronicles.Headless/GraphQLService.cs index c68027eb6..74cbbdfe5 100644 --- a/NineChronicles.Headless/GraphQLService.cs +++ b/NineChronicles.Headless/GraphQLService.cs @@ -202,11 +202,14 @@ public void ConfigureServices(IServiceCollection services) "Admin")); // FIXME: Use ConfigurationException after bumping to .NET 8 or later. - options.AddPolicy( - JwtPolicyKey, - p => - p.RequireClaim("iss", - jwtOptions["Issuer"] ?? throw new ArgumentException("jwtOptions[\"Issuer\"] is null."))); + if (Convert.ToBoolean(Configuration.GetSection("Jwt")["EnableJwtAuthentication"])) + { + options.AddPolicy( + JwtPolicyKey, + p => + p.RequireClaim("iss", + jwtOptions["Issuer"] ?? throw new ArgumentException("jwtOptions[\"Issuer\"] is null."))); + } }); services.AddGraphTypes(); @@ -220,6 +223,17 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env) } // Capture requests + app.UseMiddleware(); + + app.UseRouting(); + app.UseAuthorization(); + if (Convert.ToBoolean(Configuration.GetSection("IpRateLimiting")["EnableEndpointRateLimiting"])) + { + app.UseMiddleware(); + app.UseMiddleware(); + app.UseMvc(); + } + if (Convert.ToBoolean(Configuration.GetSection("MultiAccountManaging")["EnableManaging"])) { ConcurrentDictionary> ipSignerList = new(); @@ -229,7 +243,6 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env) Publisher); } - app.UseMiddleware(); app.UseMiddleware(); if (Convert.ToBoolean(Configuration.GetSection("Jwt")["EnableJwtAuthentication"])) @@ -246,15 +259,6 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env) app.UseCors("AllowAllOrigins"); } - app.UseRouting(); - app.UseAuthorization(); - if (Convert.ToBoolean(Configuration.GetSection("IpRateLimiting")["EnableEndpointRateLimiting"])) - { - app.UseMiddleware(); - app.UseMiddleware(); - app.UseMvc(); - } - app.UseEndpoints(endpoints => { endpoints.MapControllers(); diff --git a/NineChronicles.Headless/Middleware/CustomRateLimitMiddleware.cs b/NineChronicles.Headless/Middleware/CustomRateLimitMiddleware.cs index a1b3df76b..28b914fc3 100644 --- a/NineChronicles.Headless/Middleware/CustomRateLimitMiddleware.cs +++ b/NineChronicles.Headless/Middleware/CustomRateLimitMiddleware.cs @@ -6,32 +6,52 @@ using NineChronicles.Headless.Properties; using Serilog; using ILogger = Serilog.ILogger; +using System.Linq; +using Microsoft.Extensions.Configuration; namespace NineChronicles.Headless.Middleware { + public class CustomRateLimitMiddleware : RateLimitMiddleware { private readonly ILogger _logger; private readonly IRateLimitConfiguration _config; private readonly IOptions _options; + private readonly string _whitelistedIp; + private readonly System.IdentityModel.Tokens.Jwt.JwtSecurityTokenHandler _tokenHandler = new(); + private readonly Microsoft.IdentityModel.Tokens.TokenValidationParameters _validationParams; public CustomRateLimitMiddleware(RequestDelegate next, IProcessingStrategy processingStrategy, IOptions options, IIpPolicyStore policyStore, - IRateLimitConfiguration config) + IRateLimitConfiguration config, + Microsoft.Extensions.Configuration.IConfiguration configuration) : base(next, options?.Value, new CustomIpRateLimitProcessor(options?.Value!, policyStore, processingStrategy), config) { _config = config; _options = options!; _logger = Log.Logger.ForContext(); + var jwtConfig = configuration.GetSection("Jwt"); + var issuer = jwtConfig["Issuer"] ?? ""; + var key = jwtConfig["Key"] ?? ""; + _whitelistedIp = configuration.GetSection("IpRateLimiting:IpWhitelist")?.Get()?.FirstOrDefault() ?? "127.0.0.1"; + _validationParams = new Microsoft.IdentityModel.Tokens.TokenValidationParameters + { + ValidateIssuer = true, + ValidateAudience = false, + ValidateLifetime = true, + ValidateIssuerSigningKey = true, + ValidIssuer = issuer, + IssuerSigningKey = new Microsoft.IdentityModel.Tokens.SymmetricSecurityKey(System.Text.Encoding.ASCII.GetBytes(key.PadRight(512 / 8, '\0'))) + }; } - protected override void LogBlockedRequest(HttpContext httpContext, ClientRequestIdentity identity, RateLimitCounter counter, RateLimitRule rule) + protected override void LogBlockedRequest(HttpContext context, ClientRequestIdentity identity, RateLimitCounter counter, RateLimitRule rule) { _logger.Information($"[IP-RATE-LIMITER] Request {identity.HttpVerb}:{identity.Path} from IP {identity.ClientIp} has been blocked, " + $"quota {rule.Limit}/{rule.Period} exceeded by {counter.Count - rule.Limit}. Blocked by rule {rule.Endpoint}, " + - $"TraceIdentifier {httpContext.TraceIdentifier}. MonitorMode: {rule.MonitorMode}"); + $"TraceIdentifier {context.TraceIdentifier}. MonitorMode: {rule.MonitorMode}"); if (counter.Count - rule.Limit >= _options.Value.IpBanThresholdCount) { _logger.Information($"[IP-RATE-LIMITER] Banning IP {identity.ClientIp}."); @@ -39,23 +59,57 @@ protected override void LogBlockedRequest(HttpContext httpContext, ClientRequest } } - public override async Task ResolveIdentityAsync(HttpContext httpContext) + public override async Task ResolveIdentityAsync(HttpContext context) { - var identity = await base.ResolveIdentityAsync(httpContext); + var identity = await base.ResolveIdentityAsync(context); - if (httpContext.Request.Protocol == "HTTP/1.1") + if (context.Request.Protocol == "HTTP/1.1") { - var body = await new StreamReader(httpContext.Request.Body).ReadToEndAsync(); - httpContext.Request.Body.Seek(0, SeekOrigin.Begin); + var body = context.Items["RequestBody"]!.ToString()!; + context.Request.Body.Seek(0, SeekOrigin.Begin); if (body.Contains("stageTransaction")) { identity.Path = "/graphql/stagetransaction"; } + } - return identity; + // Check for JWT secret key in headers + if (context.Request.Headers.TryGetValue("Authorization", out var authHeaderValue) && + authHeaderValue.Count > 0) + { + try + { + var (scheme, token) = ExtractSchemeAndToken(authHeaderValue); + if (scheme.Equals("Bearer", System.StringComparison.OrdinalIgnoreCase)) + { + _tokenHandler.ValidateToken(token, _validationParams, out _); + _logger.Information("[IP-RATE-LIMITER] Valid JWT token provided. Updating ClientIp to whitelisted IP."); + identity.ClientIp = _whitelistedIp; + } + } + catch (System.Exception ex) + { + _logger.Warning("[IP-RATE-LIMITER] JWT validation failed: {Message}", ex.Message); + } } return identity; } + + private (string scheme, string token) ExtractSchemeAndToken(Microsoft.Extensions.Primitives.StringValues authorizationHeader) + { + if (authorizationHeader.Count == 0 || string.IsNullOrWhiteSpace(authorizationHeader[0])) + { + throw new System.ArgumentException("Authorization header is missing or empty."); + } + + var headerValues = authorizationHeader[0]!.Split(" "); + if (headerValues.Length != 2) + { + throw new System.ArgumentException("Invalid Authorization header format. Expected 'Scheme Token'."); + } + + return (headerValues[0], headerValues[1]); + } } } diff --git a/NineChronicles.Headless/Middleware/HttpCaptureMiddleware.cs b/NineChronicles.Headless/Middleware/HttpCaptureMiddleware.cs index f6d9c0c95..54a9b89d4 100644 --- a/NineChronicles.Headless/Middleware/HttpCaptureMiddleware.cs +++ b/NineChronicles.Headless/Middleware/HttpCaptureMiddleware.cs @@ -25,6 +25,7 @@ public async Task InvokeAsync(HttpContext context) context.Request.EnableBuffering(); var remoteIp = context.Connection.RemoteIpAddress; var body = await new StreamReader(context.Request.Body).ReadToEndAsync(); + context.Items["RequestBody"] = body; _logger.Information("[GRAPHQL-REQUEST-CAPTURE] IP: {IP} Method: {Method} Endpoint: {Path} {Body}", remoteIp, context.Request.Method, context.Request.Path, body); context.Request.Body.Seek(0, SeekOrigin.Begin); diff --git a/NineChronicles.Headless/Middleware/HttpMultiAccountManagementMiddleware.cs b/NineChronicles.Headless/Middleware/HttpMultiAccountManagementMiddleware.cs index 4fee61167..ced8cc610 100644 --- a/NineChronicles.Headless/Middleware/HttpMultiAccountManagementMiddleware.cs +++ b/NineChronicles.Headless/Middleware/HttpMultiAccountManagementMiddleware.cs @@ -27,13 +27,16 @@ public class HttpMultiAccountManagementMiddleware private readonly ConcurrentDictionary> _ipSignerList; private readonly IOptions _options; private ActionEvaluationPublisher _publisher; + private readonly System.IdentityModel.Tokens.Jwt.JwtSecurityTokenHandler _tokenHandler = new(); + private readonly Microsoft.IdentityModel.Tokens.TokenValidationParameters _validationParams; public HttpMultiAccountManagementMiddleware( RequestDelegate next, StandaloneContext standaloneContext, ConcurrentDictionary> ipSignerList, IOptions options, - ActionEvaluationPublisher publisher) + ActionEvaluationPublisher publisher, + Microsoft.Extensions.Configuration.IConfiguration configuration) { _next = next; _logger = Log.Logger.ForContext(); @@ -41,6 +44,18 @@ public HttpMultiAccountManagementMiddleware( _ipSignerList = ipSignerList; _options = options; _publisher = publisher; + var jwtConfig = configuration.GetSection("Jwt"); + var issuer = jwtConfig["Issuer"] ?? ""; + var key = jwtConfig["Key"] ?? ""; + _validationParams = new Microsoft.IdentityModel.Tokens.TokenValidationParameters + { + ValidateIssuer = true, + ValidateAudience = false, + ValidateLifetime = true, + ValidateIssuerSigningKey = true, + ValidIssuer = issuer, + IssuerSigningKey = new Microsoft.IdentityModel.Tokens.SymmetricSecurityKey(System.Text.Encoding.ASCII.GetBytes(key.PadRight(512 / 8, '\0'))) + }; } private static void ManageMultiAccount(Address agent) @@ -58,9 +73,30 @@ public async Task InvokeAsync(HttpContext context) // Prevent to harm HTTP/2 communication. if (context.Request.Protocol == "HTTP/1.1") { - context.Request.EnableBuffering(); var remoteIp = context.Connection.RemoteIpAddress!.ToString(); - var body = await new StreamReader(context.Request.Body).ReadToEndAsync(); + + // Check for JWT secret key in headers + if (context.Request.Headers.TryGetValue("Authorization", out var authHeaderValue) && + authHeaderValue.Count > 0) + { + try + { + var (scheme, token) = ExtractSchemeAndToken(authHeaderValue); + if (scheme.Equals("Bearer", System.StringComparison.OrdinalIgnoreCase)) + { + _tokenHandler.ValidateToken(token, _validationParams, out _); + _logger.Information("[GRAPHQL-MULTI-ACCOUNT-MANAGER] Valid JWT token provided. Updating ClientIp to whitelisted IP."); + await _next(context); + return; + } + } + catch (System.Exception ex) + { + _logger.Warning("[GRAPHQL-MULTI-ACCOUNT-MANAGER] JWT validation failed: {Message}", ex.Message); + } + } + + var body = context.Items["RequestBody"]!.ToString()!; context.Request.Body.Seek(0, SeekOrigin.Begin); if (_options.Value.EnableManaging && body.Contains("stageTransaction")) { @@ -150,6 +186,22 @@ and not ClaimStakeReward await _next(context); } + private (string scheme, string token) ExtractSchemeAndToken(Microsoft.Extensions.Primitives.StringValues authorizationHeader) + { + if (authorizationHeader.Count == 0 || string.IsNullOrWhiteSpace(authorizationHeader[0])) + { + throw new System.ArgumentException("Authorization header is missing or empty."); + } + + var headerValues = authorizationHeader[0]!.Split(" "); + if (headerValues.Length != 2) + { + throw new System.ArgumentException("Invalid Authorization header format. Expected 'Scheme Token'."); + } + + return (headerValues[0], headerValues[1]); + } + private void UpdateIpSignerList(string ip, Address agent) { if (!_ipSignerList.ContainsKey(ip)) From bdfda089f8d733b6c8cd7ca7f8520653f6512ebc Mon Sep 17 00:00:00 2001 From: area363 Date: Tue, 10 Dec 2024 10:42:13 +0900 Subject: [PATCH 07/10] remove unnecessary log --- .../Middleware/CustomRateLimitMiddleware.cs | 1 - .../Middleware/HttpMultiAccountManagementMiddleware.cs | 8 -------- 2 files changed, 9 deletions(-) diff --git a/NineChronicles.Headless/Middleware/CustomRateLimitMiddleware.cs b/NineChronicles.Headless/Middleware/CustomRateLimitMiddleware.cs index 28b914fc3..d390c3c44 100644 --- a/NineChronicles.Headless/Middleware/CustomRateLimitMiddleware.cs +++ b/NineChronicles.Headless/Middleware/CustomRateLimitMiddleware.cs @@ -83,7 +83,6 @@ public override async Task ResolveIdentityAsync(HttpConte if (scheme.Equals("Bearer", System.StringComparison.OrdinalIgnoreCase)) { _tokenHandler.ValidateToken(token, _validationParams, out _); - _logger.Information("[IP-RATE-LIMITER] Valid JWT token provided. Updating ClientIp to whitelisted IP."); identity.ClientIp = _whitelistedIp; } } diff --git a/NineChronicles.Headless/Middleware/HttpMultiAccountManagementMiddleware.cs b/NineChronicles.Headless/Middleware/HttpMultiAccountManagementMiddleware.cs index ced8cc610..399d3bc42 100644 --- a/NineChronicles.Headless/Middleware/HttpMultiAccountManagementMiddleware.cs +++ b/NineChronicles.Headless/Middleware/HttpMultiAccountManagementMiddleware.cs @@ -85,7 +85,6 @@ public async Task InvokeAsync(HttpContext context) if (scheme.Equals("Bearer", System.StringComparison.OrdinalIgnoreCase)) { _tokenHandler.ValidateToken(token, _validationParams, out _); - _logger.Information("[GRAPHQL-MULTI-ACCOUNT-MANAGER] Valid JWT token provided. Updating ClientIp to whitelisted IP."); await _next(context); return; } @@ -211,13 +210,6 @@ private void UpdateIpSignerList(string ip, Address agent) ip); _ipSignerList[ip] = new HashSet
(); } - else - { - _logger.Information( - "[GRAPHQL-MULTI-ACCOUNT-MANAGER] List already created for IP: {IP} Count: {Count}", - ip, - _ipSignerList[ip].Count); - } _ipSignerList[ip].Add(agent); AddClientIpInfo(agent, ip); From 24543094ae6d820135894a2efd07d3a7dc4dffb3 Mon Sep 17 00:00:00 2001 From: Atralupus Date: Tue, 17 Dec 2024 16:19:37 +0900 Subject: [PATCH 08/10] bump lib9c 84f70 --- Lib9c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib9c b/Lib9c index 0e0124be7..84f701a4d 160000 --- a/Lib9c +++ b/Lib9c @@ -1 +1 @@ -Subproject commit 0e0124be7831d85e682b534a64e16ca196c10917 +Subproject commit 84f701a4d08484a619db0113017b8d71fea8c480 From d60b66a13b83bfc9ec3987867181b9b521ca4754 Mon Sep 17 00:00:00 2001 From: area363 Date: Thu, 19 Dec 2024 12:37:02 +0900 Subject: [PATCH 09/10] add ratelimit for transactionresults --- .../appsettings.json | 6 ++ .../Middleware/CustomRateLimitMiddleware.cs | 62 +++++++++++++++++++ .../Middleware/HttpCaptureMiddleware.cs | 16 ++++- .../Middleware/IpBanMiddleware.cs | 10 +++ 4 files changed, 92 insertions(+), 2 deletions(-) diff --git a/NineChronicles.Headless.Executable/appsettings.json b/NineChronicles.Headless.Executable/appsettings.json index 3e4d70044..8f42ab9db 100644 --- a/NineChronicles.Headless.Executable/appsettings.json +++ b/NineChronicles.Headless.Executable/appsettings.json @@ -109,6 +109,11 @@ "Endpoint": "*:/graphql/stagetransaction", "Period": "60s", "Limit": 12 + }, + { + "Endpoint": "*:/graphql/transactionresults", + "Period": "60s", + "Limit": 60 } ], "QuotaExceededResponse": { @@ -117,6 +122,7 @@ "StatusCode": 429 }, "IpBanThresholdCount": 5, + "TransactionResultsBanThresholdCount": 100, "IpBanMinute" : 60, "IpBanResponse": { "Content": "{ \"message\": \"Your Ip has been banned.\" }", diff --git a/NineChronicles.Headless/Middleware/CustomRateLimitMiddleware.cs b/NineChronicles.Headless/Middleware/CustomRateLimitMiddleware.cs index d390c3c44..c86457be0 100644 --- a/NineChronicles.Headless/Middleware/CustomRateLimitMiddleware.cs +++ b/NineChronicles.Headless/Middleware/CustomRateLimitMiddleware.cs @@ -18,6 +18,7 @@ public class CustomRateLimitMiddleware : RateLimitMiddleware _options; private readonly string _whitelistedIp; + private readonly int _banCount; private readonly System.IdentityModel.Tokens.Jwt.JwtSecurityTokenHandler _tokenHandler = new(); private readonly Microsoft.IdentityModel.Tokens.TokenValidationParameters _validationParams; @@ -36,6 +37,7 @@ public CustomRateLimitMiddleware(RequestDelegate next, var issuer = jwtConfig["Issuer"] ?? ""; var key = jwtConfig["Key"] ?? ""; _whitelistedIp = configuration.GetSection("IpRateLimiting:IpWhitelist")?.Get()?.FirstOrDefault() ?? "127.0.0.1"; + _banCount = configuration.GetValue("IpRateLimiting:TransactionResultsBanThresholdCount", 100); _validationParams = new Microsoft.IdentityModel.Tokens.TokenValidationParameters { ValidateIssuer = true, @@ -71,6 +73,18 @@ public override async Task ResolveIdentityAsync(HttpConte { identity.Path = "/graphql/stagetransaction"; } + else if (body.Contains("transactionResults")) + { + identity.Path = "/graphql/transactionresults"; + + // Check for txIds count + var txIdsCount = CountTxIds(body); + if (txIdsCount > _banCount) + { + _logger.Information($"[IP-RATE-LIMITER] Banning IP {identity.ClientIp} due to excessive txIds count: {txIdsCount}"); + IpBanMiddleware.BanIp(identity.ClientIp); + } + } } // Check for JWT secret key in headers @@ -110,5 +124,53 @@ public override async Task ResolveIdentityAsync(HttpConte return (headerValues[0], headerValues[1]); } + + private int CountTxIds(string body) + { + try + { + var json = System.Text.Json.JsonDocument.Parse(body); + + // Check for txIds in query variables first + if (json.RootElement.TryGetProperty("variables", out var variables) && + variables.TryGetProperty("txIds", out var txIdsElement) && + txIdsElement.ValueKind == System.Text.Json.JsonValueKind.Array) + { + // Count from variables + return txIdsElement.GetArrayLength(); + } + + // Fallback to check the query string if variables are not set + if (json.RootElement.TryGetProperty("query", out var queryElement)) + { + var query = queryElement.GetString(); + if (!string.IsNullOrWhiteSpace(query)) + { + // Extract txIds from the query string using regex + var txIdMatches = System.Text.RegularExpressions.Regex.Matches( + query, @"transactionResults\s*\(\s*txIds\s*:\s*\[(?[^\]]*)\]" + ); + + if (txIdMatches.Count > 0) + { + // Extract the inner contents of txIds + var txIdList = txIdMatches[0].Groups["txIds"].Value; + + // Count txIds using commas + var txIds = txIdList.Split(',', System.StringSplitOptions.RemoveEmptyEntries | System.StringSplitOptions.TrimEntries); + + return txIds.Length; + } + } + } + } + catch (System.Exception ex) + { + _logger.Warning("[IP-RATE-LIMITER] Error parsing request body: {Message}", ex.Message); + } + + // Return 0 if txIds not found + return 0; + } } } diff --git a/NineChronicles.Headless/Middleware/HttpCaptureMiddleware.cs b/NineChronicles.Headless/Middleware/HttpCaptureMiddleware.cs index 54a9b89d4..a83f9745e 100644 --- a/NineChronicles.Headless/Middleware/HttpCaptureMiddleware.cs +++ b/NineChronicles.Headless/Middleware/HttpCaptureMiddleware.cs @@ -3,6 +3,7 @@ using Microsoft.AspNetCore.Http; using Serilog; using ILogger = Serilog.ILogger; +using Microsoft.Extensions.Configuration; namespace NineChronicles.Headless.Middleware { @@ -10,20 +11,31 @@ public class HttpCaptureMiddleware { private readonly RequestDelegate _next; private readonly ILogger _logger; + private readonly bool _enableIpRateLimiting; - public HttpCaptureMiddleware(RequestDelegate next) + public HttpCaptureMiddleware(RequestDelegate next, Microsoft.Extensions.Configuration.IConfiguration configuration) { _next = next; _logger = Log.Logger.ForContext(); + _enableIpRateLimiting = configuration.GetValue("IpRateLimiting:EnableEndpointRateLimiting"); } public async Task InvokeAsync(HttpContext context) { + var remoteIp = context.Connection.RemoteIpAddress!.ToString(); + + // Conditionally skip IP banning if endpoint rate-limiting is disabled + if (_enableIpRateLimiting && IpBanMiddleware.IsIpBanned(remoteIp)) + { + _logger.Information($"[GRAPHQL-REQUEST-CAPTURE] Skipping logging for banned IP: {remoteIp}"); + await _next(context); + return; + } + // Prevent to harm HTTP/2 communication. if (context.Request.Protocol == "HTTP/1.1") { context.Request.EnableBuffering(); - var remoteIp = context.Connection.RemoteIpAddress; var body = await new StreamReader(context.Request.Body).ReadToEndAsync(); context.Items["RequestBody"] = body; _logger.Information("[GRAPHQL-REQUEST-CAPTURE] IP: {IP} Method: {Method} Endpoint: {Path} {Body}", diff --git a/NineChronicles.Headless/Middleware/IpBanMiddleware.cs b/NineChronicles.Headless/Middleware/IpBanMiddleware.cs index 8f391e56e..6210cd39d 100644 --- a/NineChronicles.Headless/Middleware/IpBanMiddleware.cs +++ b/NineChronicles.Headless/Middleware/IpBanMiddleware.cs @@ -39,6 +39,16 @@ public static void UnbanIp(string ip) } } + public static bool IsIpBanned(string ip) + { + if (_bannedIps.ContainsKey(ip)) + { + return true; + } + + return false; + } + public Task InvokeAsync(HttpContext context) { var remoteIp = context.Connection.RemoteIpAddress!.ToString(); From baf375c1e8f0085fcd6fbb68e0ce3fa3a1dd6ec9 Mon Sep 17 00:00:00 2001 From: Atralupus Date: Fri, 20 Dec 2024 00:47:46 +0900 Subject: [PATCH 10/10] bump lib9c --- Lib9c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib9c b/Lib9c index 84f701a4d..e639502af 160000 --- a/Lib9c +++ b/Lib9c @@ -1 +1 @@ -Subproject commit 84f701a4d08484a619db0113017b8d71fea8c480 +Subproject commit e639502afb22df4d230d52002b519b3f8ef16b4f