Raft fixes + ASP.NET Plugin that will redirect HTTP requests to the l…

…eader #20

Signed-off-by: Tomasz Maruszak <[email protected]>

README.md

@@ -44,6 +44,7 @@ The path to a stable production release:
 | `SlimCluster.Serialization.Json`    | JSON message serialization plugin          | [![NuGet](https://img.shields.io/nuget/v/SlimCluster.Serialization.Json.svg)](https://www.nuget.org/packages/SlimCluster.Serialization.Json)       |
 | `SlimCluster.Transport.Ip`          | IP protocol transport plugin               | [![NuGet](https://img.shields.io/nuget/v/SlimCluster.Transport.Ip.svg)](https://www.nuget.org/packages/SlimCluster.Transport.Ip)                   |
 | `SlimCluster.Persistence.LocalFile` | Persists node state into a local JSON file | [![NuGet](https://img.shields.io/nuget/v/SlimCluster.Persistence.LocalFile.svg)](https://www.nuget.org/packages/SlimCluster.Persistence.LocalFile) |
+| `SlimCluster.AspNetCore`            | ASP.NET request routing to Leader node     | [![NuGet](https://img.shields.io/nuget/v/SlimCluster.AspNetCore.svg)](https://www.nuget.org/packages/SlimCluster.AspNetCore)                       |
 
 ## Samples
 
@@ -77,6 +78,12 @@ builder.Services.AddSlimCluster(cfg =>
     cfg.AddRaftConsensus(opts =>
     {
         opts.NodeCount = 3;
+
+        // Use custom values or remove and use defaults
+        opts.LeaderTimeout = TimeSpan.FromSeconds(5);
+        opts.LeaderPingInterval = TimeSpan.FromSeconds(2);
+        opts.ElectionTimeoutMin = TimeSpan.FromSeconds(3);
+        opts.ElectionTimeoutMax = TimeSpan.FromSeconds(6);
         // Can set a different log serializer, by default ISerializer is used (in our setup its JSON)
         // opts.LogSerializerType = typeof(JsonSerializer);
     });
@@ -86,6 +93,12 @@ builder.Services.AddSlimCluster(cfg =>
 
     // Cluster state will saved into the local json file in between node restarts
     cfg.AddPersistenceUsingLocalFile("cluster-state.json");
+
+    cfg.AddAspNetCore(opts =>
+    {
+        // Route all ASP.NET API requests for the Counter endpoint to the Leader node for handling
+        opts.DelegateRequestToLeader = r => r.Path.HasValue && r.Path.Value.Contains("/Counter");
+    });
 });
 
 // Raft app specific implementation

README.t.md

@@ -44,6 +44,7 @@ The path to a stable production release:
 | `SlimCluster.Serialization.Json`    | JSON message serialization plugin          | [![NuGet](https://img.shields.io/nuget/v/SlimCluster.Serialization.Json.svg)](https://www.nuget.org/packages/SlimCluster.Serialization.Json)       |
 | `SlimCluster.Transport.Ip`          | IP protocol transport plugin               | [![NuGet](https://img.shields.io/nuget/v/SlimCluster.Transport.Ip.svg)](https://www.nuget.org/packages/SlimCluster.Transport.Ip)                   |
 | `SlimCluster.Persistence.LocalFile` | Persists node state into a local JSON file | [![NuGet](https://img.shields.io/nuget/v/SlimCluster.Persistence.LocalFile.svg)](https://www.nuget.org/packages/SlimCluster.Persistence.LocalFile) |
+| `SlimCluster.AspNetCore`            | ASP.NET request routing to Leader node     | [![NuGet](https://img.shields.io/nuget/v/SlimCluster.AspNetCore.svg)](https://www.nuget.org/packages/SlimCluster.AspNetCore)                       |
 
 ## Samples
 

build/tasks.ps1

@@ -26,6 +26,8 @@ $projects = @(
 
 	"SlimCluster.Transport",
 	"SlimCluster.Transport.Ip"
+
+	"SlimCluster.AspNetCore",
 )
 
 # msbuild.exe https://msdn.microsoft.com/pl-pl/library/ms164311(v=vs.80).aspx

src/.editorconfig

@@ -102,6 +102,7 @@ csharp_style_prefer_readonly_struct = true:suggestion
 csharp_style_allow_embedded_statements_on_same_line_experimental = true:silent
 csharp_style_allow_blank_lines_between_consecutive_braces_experimental = true:silent
 csharp_style_allow_blank_line_after_colon_in_constructor_initializer_experimental = true:silent
+csharp_style_prefer_readonly_struct_member = true:suggestion
 
 [*.{cs,vb}]
 #### Naming styles ####
@@ -120,6 +121,16 @@ dotnet_naming_rule.non_field_members_should_be_pascal_case.severity = suggestion
 dotnet_naming_rule.non_field_members_should_be_pascal_case.symbols = non_field_members
 dotnet_naming_rule.non_field_members_should_be_pascal_case.style = pascal_case
 
+dotnet_naming_rule.private_members_with_underscore.symbols  = private_fields
+dotnet_naming_rule.private_members_with_underscore.style    = prefix_underscore
+dotnet_naming_rule.private_members_with_underscore.severity = suggestion
+
+dotnet_naming_symbols.private_fields.applicable_kinds           = field
+dotnet_naming_symbols.private_fields.applicable_accessibilities = private
+
+dotnet_naming_style.prefix_underscore.capitalization = camel_case
+dotnet_naming_style.prefix_underscore.required_prefix = _
+
 # Symbol specifications
 
 dotnet_naming_symbols.interface.applicable_kinds = interface

src/Common.NuGet.Properties.xml

@@ -3,7 +3,7 @@
   <Import Project="Common.Properties.xml" />
 
   <PropertyGroup>
-    <Version>0.8.1</Version>
+    <Version>0.9.0</Version>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>
     <PackageReadmeFile>NuGet.md</PackageReadmeFile>
     <PackageIcon>icon.png</PackageIcon>

src/Samples/Dockerfile

@@ -8,10 +8,10 @@ COPY docs ./docs
 WORKDIR /source/src
 ARG SERVICE
 RUN dotnet publish "Samples/${SERVICE}" -c Release -o /app/publish
+RUN mv /app/publish/${SERVICE}.dll /app/publish/MainService.dll && \
+    mv /app/publish/${SERVICE}.runtimeconfig.json /app/publish/MainService.runtimeconfig.json
 
 FROM base AS final
 WORKDIR /app
 COPY --from=build /app/publish .
-ARG SERVICE
-ENV SERVICE_FILE "${SERVICE}.dll"
-ENTRYPOINT [ "/usr/bin/dotnet", "SlimCluster.Samples.Service.dll" ]
+ENTRYPOINT [ "/usr/bin/dotnet", "MainService.dll" ]

src/Samples/SlimCluster.Samples.Service/Controllers/CounterController.cs

@@ -20,23 +20,27 @@ public CounterController(ICounterState counterState, IRaftClientRequestHandler c
     }
 
     [HttpGet()]
+    [ProducesResponseType(typeof(int), StatusCodes.Status200OK)]
     public int Get() => _counterState.Counter;
 
     [HttpPost("[action]")]
+    [ProducesResponseType(typeof(int), StatusCodes.Status200OK)]
     public async Task<int?> Increment(CancellationToken cancellationToken)
     {
         var result = await _clientRequestHandler.OnClientRequest(new IncrementCounterCommand(), cancellationToken);
         return (int?)result;
     }
 
     [HttpPost("[action]")]
+    [ProducesResponseType(typeof(int), StatusCodes.Status200OK)]
     public async Task<int?> Decrement(CancellationToken cancellationToken)
     {
         var result = await _clientRequestHandler.OnClientRequest(new DecrementCounterCommand(), cancellationToken);
         return (int?)result;
     }
 
     [HttpPost("[action]")]
+    [ProducesResponseType(typeof(int), StatusCodes.Status200OK)]
     public async Task<int?> Reset(CancellationToken cancellationToken)
     {
         var result = await _clientRequestHandler.OnClientRequest(new ResetCounterCommand(), cancellationToken);

src/Samples/SlimCluster.Samples.Service/Kube-DeleteSample.ps1

@@ -1,2 +1,2 @@
 kubectl delete -f deployment.yaml
-kubectl delete -f service.yaml
+#kubectl delete -f service.yaml

src/Samples/SlimCluster.Samples.Service/Program.cs

@@ -1,4 +1,5 @@
 using SlimCluster;
+using SlimCluster.AspNetCore;
 using SlimCluster.Consensus.Raft;
 using SlimCluster.Consensus.Raft.Logs;
 using SlimCluster.Membership.Swim;
@@ -12,8 +13,6 @@
 
 var builder = WebApplication.CreateBuilder(args);
 
-//builder.Host.UseConsoleLifetime();
-
 builder.Services.AddControllers();
 builder.Services.AddEndpointsApiExplorer();
 builder.Services.AddSwaggerGen();
@@ -44,6 +43,12 @@
     cfg.AddRaftConsensus(opts =>
     {
         opts.NodeCount = 3;
+
+        // Use custom values or remove and use defaults
+        opts.LeaderTimeout = TimeSpan.FromSeconds(5);
+        opts.LeaderPingInterval = TimeSpan.FromSeconds(2);
+        opts.ElectionTimeoutMin = TimeSpan.FromSeconds(3);
+        opts.ElectionTimeoutMax = TimeSpan.FromSeconds(6);
         // Can set a different log serializer, by default ISerializer is used (in our setup its JSON)
         // opts.LogSerializerType = typeof(JsonSerializer);
     });
@@ -53,6 +58,12 @@
 
     // Cluster state will saved into the local json file in between node restarts
     cfg.AddPersistenceUsingLocalFile("cluster-state.json");
+
+    cfg.AddAspNetCore(opts =>
+    {
+        // Route all ASP.NET API requests for the Counter endpoint to the Leader node for handling
+        opts.DelegateRequestToLeader = r => r.Path.HasValue && r.Path.Value.Contains("/Counter");
+    });
 });
 
 // Raft app specific implementation
@@ -78,6 +89,9 @@
 
 app.UseAuthorization();
 
+// Delegate selected ASP.NET API requests to the leader node for handling
+app.UseClusterLeaderRequestDelegation();
+
 app.MapControllers();
 
 await app.RunAsync();

src/Samples/SlimCluster.Samples.Service/SlimCluster.Samples.Service.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk.Web">
+<Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
     <TargetFramework>net7.0</TargetFramework>
@@ -12,6 +12,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\..\SlimCluster.AspNetCore\SlimCluster.AspNetCore.csproj" />
     <ProjectReference Include="..\..\SlimCluster.Membership.Swim\SlimCluster.Membership.Swim.csproj" />
     <ProjectReference Include="..\..\SlimCluster.Serialization.Json\SlimCluster.Serialization.Json.csproj" />
     <ProjectReference Include="..\..\SlimCluster.Consensus.Raft\SlimCluster.Consensus.Raft.csproj" />

src/Samples/SlimCluster.Samples.Service/State/Logs/AbstractCommand.cs

@@ -1,4 +1,5 @@
 namespace SlimCluster.Samples.ConsoleApp.State.Logs;
+
 public abstract record AbstractCommand
 {
 }

src/Samples/SlimCluster.Samples.Service/State/StateMachine/CounterStateMachine.cs

@@ -8,10 +8,10 @@
 /// </summary>
 public class CounterStateMachine : IStateMachine, ICounterState
 {
-    private int _intex = 0;
+    private int _index = 0;
     private int _counter = 0;
 
-    public int CurrentIndex => _intex;
+    public int CurrentIndex => _index;
 
     /// <summary>
     /// The counter value
@@ -22,9 +22,9 @@ public class CounterStateMachine : IStateMachine, ICounterState
     {
         // Note: This is thread safe - there is ever going to be only one task at a time calling Apply
 
-        if (_intex + 1 != index)
+        if (_index + 1 != index)
         {
-            throw new InvalidOperationException($"The State Machine can only apply next command at index ${_intex + 1}");
+            throw new InvalidOperationException($"The State Machine can only apply next command at index ${_index + 1}");
         }
 
         int? result = command switch
@@ -35,7 +35,7 @@ public class CounterStateMachine : IStateMachine, ICounterState
             _ => throw new NotImplementedException($"The command type ${command?.GetType().Name} is not supported")
         };
 
-        _intex = index;
+        _index = index;
 
         return Task.FromResult<object?>(result);
     }

src/Samples/SlimCluster.Samples.Service/appsettings.json

@@ -5,7 +5,25 @@
       "SlimCluster": "Information",
       "SlimCluster.Transport": "Information",
       "SlimCluster.Consensus": "Information",
-      "SlimCluster.Membership": "Information"
+      "SlimCluster.Membership": "Information",
+      "SlimCluster.Samples.Service.Middleware.RequestDelegatingClient": "Information",
+      "Microsoft.AspNetCore.Mvc": "Warning",
+      "Microsoft.AspNetCore.Hosting.Diagnostics": "Warning",
+      "Microsoft.AspNetCore.Routing": "Warning"
+    },
+    "Console": {
+      "LogLevel": {
+      },
+      "FormatterName": "CustomTimePrefixingFormatter",
+      "FormatterOptions": {
+        "SingleLine": false,
+        "IncludeScopes": true,
+        "TimestampFormat": "HH:mm:ss.ffff ",
+        "UseUtcTimestamp": true,
+        "JsonWriterOptions": {
+          "Indented": true
+        }
+      }
     }
   },
   "UdpPort": "60001",

src/Samples/SlimCluster.Samples.Service/deployment.yaml

@@ -20,8 +20,8 @@ spec:
         image: zarusz/slimcluster_samples_service:latest
         imagePullPolicy: IfNotPresent # ensure we get always the fresh container image build
         ports:
-        - containerPort: 5000
+        - containerPort: 8080
         - containerPort: 60001
         env:
         - name: ASPNETCORE_URLS
-          value: "http://+:5000"
+          value: "http://+:8080"

src/Samples/SlimCluster.Samples.Service/service.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   type: NodePort
   ports:
-  - port: 5000
+  - port: 8080
     protocol: TCP
   selector:
     run: sc-service

src/SlimCluster.AspNetCore/Configuration/ClusterAspNetOptions.cs

@@ -0,0 +1,12 @@
+namespace SlimCluster.Consensus.Raft;
+
+using Microsoft.AspNetCore.Http;
+
+public class ClusterAspNetOptions
+{
+    /// <summary>
+    /// Selects the request to be routed to the Leader node of the cluster. When not set then no requests are being routed to leader.
+    /// </summary>
+    public Func<HttpRequest, bool>? DelegateRequestToLeader { get; set; }
+
+}

src/SlimCluster.AspNetCore/Configuration/ClusterConfigurationExtensions.cs

@@ -0,0 +1,19 @@
+namespace SlimCluster.Consensus.Raft;
+
+using Microsoft.Extensions.DependencyInjection;
+
+using SlimCluster.AspNetCore;
+
+public static class ClusterConfigurationExtensions
+{
+    public static ClusterConfiguration AddAspNetCore(this ClusterConfiguration cfg, Action<ClusterAspNetOptions> options)
+    {
+        cfg.PostConfigurationActions.Add(services =>
+        {
+            services.AddHttpClient<RequestDelegatingClient>();
+
+            services.Configure(options);
+        });
+        return cfg;
+    }
+}

src/SlimCluster.AspNetCore/Middleware/ClusterLeaderRequestDelegatingMiddleware.cs

@@ -0,0 +1,44 @@
+namespace SlimCluster.AspNetCore;
+
+using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.Options;
+
+using SlimCluster.Consensus.Raft;
+
+public class ClusterLeaderRequestDelegatingMiddleware
+{
+    private readonly RequestDelegate _next;
+    private readonly ICluster _cluster;
+    private readonly RequestDelegatingClient _requestDelegatingClient;
+    private readonly ClusterAspNetOptions _options;
+
+    public ClusterLeaderRequestDelegatingMiddleware(RequestDelegate next, ICluster cluster, RequestDelegatingClient requestDelegatingClient, IOptions<ClusterAspNetOptions> options)
+    {
+        _next = next;
+        _cluster = cluster;
+        _requestDelegatingClient = requestDelegatingClient;
+        _options = options.Value;
+    }
+
+    public async Task InvokeAsync(HttpContext context)
+    {
+        // Check if request should be routed to leader
+        if (_options.DelegateRequestToLeader != null && _options.DelegateRequestToLeader(context.Request))
+        {
+            var leaderAddress = _cluster.LeaderNode?.Address;
+            if (leaderAddress == null)
+            {
+                throw new ClusterException("Leader not known at this time, retry request later on when leader is established");
+            }
+
+            if (!_cluster.SelfNode.Equals(_cluster.LeaderNode))
+            {
+                // This is a follower, so need to delegate the call to the leader
+                await _requestDelegatingClient.Delegate(context.Request, context.Response, leaderAddress, localPort: context.Connection.LocalPort);
+                return;
+            }
+        }
+        // Not subject for routing, or this is the leader node (safe to pass the request processing by self)
+        await _next(context);
+    }
+}

src/SlimCluster.AspNetCore/Middleware/ClusterLeaderRequestDelegatingMiddlewareExtensions.cs

@@ -0,0 +1,14 @@
+namespace SlimCluster.AspNetCore;
+
+using Microsoft.AspNetCore.Builder;
+
+public static class ClusterLeaderRequestDelegatingMiddlewareExtensions
+{
+    /// <summary>
+    /// Use the Request Delegation to Leader node (if self is not a leader).
+    /// </summary>
+    /// <param name="builder"></param>
+    /// <returns></returns>
+    public static IApplicationBuilder UseClusterLeaderRequestDelegation(this IApplicationBuilder builder)
+        => builder.UseMiddleware<ClusterLeaderRequestDelegatingMiddleware>();
+}