New event appending performance harness

src/DaemonTests/TestingSupport/TripStream.cs

@@ -2,10 +2,20 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
+using System.Threading.Tasks;
+using JasperFx.Core;
+using Marten;
 using Marten.Events;
 
 namespace DaemonTests.TestingSupport;
 
+public class TripPublisher
+{
+    public List<object> Events { get; set; } = new();
+
+
+}
+
 public class TripStream
 {
     public static List<TripStream> RandomStreams(int number)
@@ -50,11 +60,14 @@ public static TimeOnly RandomTime()
         return new TimeOnly(hour, 0, 0);
     }
 
-    public Guid StreamId = Guid.NewGuid();
+    public Guid StreamId = CombGuidIdGeneration.NewGuid();
 
     public readonly List<object> Events = new List<object>();
 
-
+    public TripStream(List<object> events)
+    {
+        Events = events;
+    }
 
     public TripStream()
     {
@@ -125,6 +138,91 @@ public void Reset()
         _index = 0;
     }
 
+    public static async Task PublishMultiplesSimple(IDocumentSession session, TripStream[] streams)
+    {
+        var list = streams.ToList();
+        while (list.Any())
+        {
+            foreach (var stream in list.ToArray())
+            {
+                if (stream.TryCheckOutEvents(out var events))
+                {
+                    if (stream.Started)
+                    {
+                        session.Events.Append(stream.StreamId, events);
+                    }
+                    else
+                    {
+                        stream.Started = true;
+                        session.Events.StartStream<Trip>(stream.StreamId, events);
+                    }
+                }
+
+                if (stream.IsFinishedPublishing())
+                {
+                    list.Remove(stream);
+                }
+            }
+
+            await session.SaveChangesAsync();
+        }
+    }
+
+    public bool Started { get; private set; }
+
+    public async Task PublishSingleFileSimple(IDocumentSession session)
+    {
+        while (TryAppendSimple(session))
+        {
+            await session.SaveChangesAsync();
+        }
+    }
+
+    public async Task PublishSingleFileWithFetchForWriting(IDocumentSession session)
+    {
+        while (await TryAppendWithFetchForWriting(session))
+        {
+            await session.SaveChangesAsync();
+        }
+    }
+
+    public async Task<bool> TryAppendWithFetchForWriting(IDocumentSession session)
+    {
+        if (TryCheckOutEvents(out var events))
+        {
+            if (Started)
+            {
+                var stream = await session.Events.FetchForWriting<Trip>(StreamId);
+                stream.AppendMany(events);
+            }
+            else
+            {
+                session.Events.StartStream<Trip>(StreamId, events);
+                Started = true;
+            }
+        }
+
+        return !IsFinishedPublishing();
+    }
+
+    public bool TryAppendSimple(IDocumentSession session)
+    {
+        if (TryCheckOutEvents(out var events))
+        {
+            if (Started)
+            {
+                session.Events.Append(StreamId, events);
+            }
+            else
+            {
+                session.Events.StartStream<Trip>(StreamId, events);
+                Started = true;
+            }
+        }
+
+        return !IsFinishedPublishing();
+    }
+
     public bool TryCheckOutEvents(out object[] events)
     {
         if (IsFinishedPublishing())

src/EventAppenderPerfTester/EventAppenderPerfTester.csproj

@@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <OutputType>Exe</OutputType>
+        <TargetFramework>net8.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\DaemonTests\DaemonTests.csproj" />
+      <ProjectReference Include="..\Marten.CommandLine\Marten.CommandLine.csproj" />
+    </ItemGroup>
+
+</Project>

src/EventAppenderPerfTester/ExportCommand.cs

@@ -0,0 +1,25 @@
+using System.Runtime.CompilerServices;
+using DaemonTests.TestingSupport;
+using Oakton;
+using Xunit.Abstractions;
+
+namespace EventAppenderPerfTester;
+
+public class ExportCommand : OaktonAsyncCommand<NetCoreInput>
+{
+    public override async Task<bool> Execute(NetCoreInput input)
+    {
+        using var host = input.BuildHost();
+        var trips = new List<TripStream>();
+        for (int i = 0; i < 100; i++)
+        {
+            trips.Add(new TripStream());
+        }
+
+        await TripStreamReaderWriter.Write(trips.ToArray());
+
+        Console.WriteLine("Wrote 100 trip streams to " + TripStreamReaderWriter.Path);
+
+        return true;
+    }
+}

src/EventAppenderPerfTester/ITestPlan.cs

@@ -0,0 +1,9 @@
+using Marten;
+
+namespace EventAppenderPerfTester;
+
+public interface ITestPlan
+{
+    Task Execute(IDocumentSession session);
+    void FetchData();
+}

src/EventAppenderPerfTester/MultiplesTestPlan.cs

@@ -0,0 +1,22 @@
+using DaemonTests.TestingSupport;
+using Marten;
+
+namespace EventAppenderPerfTester;
+
+public class MultiplesTestPlan: ITestPlan
+{
+    private List<TripStream[]> _data;
+
+    public async Task Execute(IDocumentSession session)
+    {
+        foreach (var trips in _data)
+        {
+            await TripStream.PublishMultiplesSimple(session, trips);
+        }
+    }
+
+    public void FetchData()
+    {
+        _data = TripStreamReaderWriter.ReadPages(100, 10);
+    }
+}

src/EventAppenderPerfTester/Program.cs

@@ -0,0 +1,25 @@
+using Marten;
+using Marten.Events;
+using Marten.Events.Projections;
+using Marten.Testing.Harness;
+using Microsoft.Extensions.Hosting;
+using Oakton;
+
+var builder = Host.CreateDefaultBuilder();
+builder.ConfigureServices(services =>
+{
+    services.AddMarten(opts =>
+    {
+        opts.Connection(ConnectionSource.ConnectionString);
+
+        opts.DisableNpgsqlLogging = true;
+
+        opts.Events.AppendMode = EventAppendMode.Quick;
+        opts.Events.UseIdentityMapForInlineAggregates = true;
+
+        opts.Projections.Add<DaemonTests.TestingSupport.TripProjection>(ProjectionLifecycle.Inline);
+    });
+});
+
+return await builder.RunOaktonCommands(args);
+

src/EventAppenderPerfTester/ReadCommand.cs

@@ -0,0 +1,16 @@
+using Oakton;
+
+namespace EventAppenderPerfTester;
+
+public class ReadCommand : OaktonAsyncCommand<NetCoreInput>
+{
+    public override async Task<bool> Execute(NetCoreInput input)
+    {
+        using var host = input.BuildHost();
+        var trips = TripStreamReaderWriter.Read();
+
+        Console.WriteLine("Read trips");
+
+        return true;
+    }
+}

src/EventAppenderPerfTester/Results.md

@@ -0,0 +1,40 @@
+# Testing Results
+
+Baseline:
+
+```
+┌───────────────────────────┬──────────────────────────┐
+│ Test Name                 │ Duration in milliseconds │
+├───────────────────────────┼──────────────────────────┤
+│ SingleFileSimple          │                     5502 │
+│ SingleFileFetchForWriting │                     5031 │
+│ Multiples                 │                    27929 │
+└───────────────────────────┴──────────────────────────┘
+
+```
+
+Adding Quick Append:
+
+```
+┌───────────────────────────┬──────────────────────────┐
+│ Test Name                 │ Duration in milliseconds │
+├───────────────────────────┼──────────────────────────┤
+│ SingleFileSimple          │                     4517 │
+│ SingleFileFetchForWriting │                     3338 │
+│ Multiples                 │                    14722 │
+└───────────────────────────┴──────────────────────────┘
+
+```
+
+Quick Append and the Inline + FetchForWriting thing:
+
+```
+┌───────────────────────────┬──────────────────────────┐
+│ Test Name                 │ Duration in milliseconds │
+├───────────────────────────┼──────────────────────────┤
+│ SingleFileSimple          │                     4234 │
+│ SingleFileFetchForWriting │                     3054 │
+│ Multiples                 │                    15171 │
+└───────────────────────────┴──────────────────────────┘
+
+```

src/EventAppenderPerfTester/SingleFileFetchForWritingPlan.cs

@@ -0,0 +1,22 @@
+using DaemonTests.TestingSupport;
+using Marten;
+
+namespace EventAppenderPerfTester;
+
+public class SingleFileFetchForWritingPlan: ITestPlan
+{
+    private List<TripStream> _data;
+
+    public async Task Execute(IDocumentSession session)
+    {
+        foreach (var tripStream in _data)
+        {
+            await tripStream.PublishSingleFileWithFetchForWriting(session);
+        }
+    }
+
+    public void FetchData()
+    {
+        _data = TripStreamReaderWriter.ReadPages(1);
+    }
+}

src/EventAppenderPerfTester/SingleFileSimplePlan.cs

@@ -0,0 +1,22 @@
+using DaemonTests.TestingSupport;
+using Marten;
+
+namespace EventAppenderPerfTester;
+
+public class SingleFileSimplePlan: ITestPlan
+{
+    private List<TripStream> _data;
+
+    public async Task Execute(IDocumentSession session)
+    {
+        foreach (var tripStream in _data)
+        {
+            await tripStream.PublishSingleFileSimple(session);
+        }
+    }
+
+    public void FetchData()
+    {
+        _data = TripStreamReaderWriter.ReadPages(1);
+    }
+}