New query plan feature. Closes GH-3318

docs/.vitepress/config.mts

@@ -125,7 +125,7 @@ const config: UserConfig<DefaultTheme.Config> = {
                 { text: 'Querying with Postgres SQL', link: '/documents/querying/sql' },
                 { text: 'Advanced Querying with Postgres SQL', link: '/documents/querying/advanced-sql' },
                 { text: 'Querying for Raw JSON', link: '/documents/querying/query-json' },
-                { text: 'Compiled Queries', link: '/documents/querying/compiled-queries' },
+                { text: 'Compiled Queries and Reusable Query Plans', link: '/documents/querying/compiled-queries' },
                 { text: 'Batched Queries', link: '/documents/querying/batched-queries' },]
             },
 

docs/documents/querying/compiled-queries.md

@@ -1,9 +1,13 @@
-# Compiled Queries
+# Compiled Queries and Query Plans
 
-::: tip
-The compiled query support was completely rewritten for Marten V4, and the signature changed somewhat. The new signature depends
-on `IMartenQueryable<T>` instead of `IQueryable<T>`, and most Marten specific Linq usages are available.
-:::
+Marten has two different implementations for the ["Specification" pattern](https://deviq.com/design-patterns/specification-pattern) that
+enable you to encapsulate all the filtering, ordering, and paging for a logically reusable data query into a single class:
+
+1. Compiled queries that help short circuit the LINQ processing with reusable "execution plans" for maximum performance, but
+   are admittedly limited in terms of their ability to handle all possible LINQ queries or basically any dynamic querying whatsoever
+2. Query plans that can support anything that Marten itself can do, just without the magic LINQ query compilation optimization
+
+## Compiled Queries
 
 ::: warning
 Don't use asynchronous Linq operators in the expression body of a compiled query. This will not impact your ability to use compiled queries
@@ -594,3 +598,41 @@ public async Task use_compiled_query_with_statistics()
 ```
 <sup><a href='https://github.com/JasperFx/marten/blob/master/src/LinqTests/Compiled/compiled_queries.cs#L37-L55' title='Snippet source file'>snippet source</a> | <a href='#snippet-sample_using_querystatistics_with_compiled_query' title='Start of snippet'>anchor</a></sup>
 <!-- endSnippet -->
+
+## Query Plans <Badge type="tip" text="7.25" />
+
+::: info
+The query plan concept was created specifically to help a [JasperFx](https://jasperfx.net) client try to eliminate their
+custom repository wrappers around Marten and to better utilize batch querying.
+:::
+
+::: tip
+Batch querying is a great way to improve the performance of your system~~~~
+:::
+
+A query plan is another flavor of "Specification" for Marten that just enables you to bundle up query logic that can
+be reused within your codebase without having to create wrappers around Marten itself. To create a reusable query plan,
+implement the `IQueryPlan<T>` interface where `T` is the type of the result you want. Here's a simplistic sample from
+the tests:
+
+snippet: sample_color_targets
+
+And then use that like so:
+
+snippet: sample_using_query_plan
+
+There is also a similar interface for usage with [batch querying](/documents/querying/batched-queries):
+
+snippet: sample_IBatchQueryPlan
+
+And because we expect this to be very common, there is convenience base class named `QueryListPlan<T>` for querying lists of 
+`T` data that can be used for both querying directly against an `IQuerySession` and for batch querying. The usage within
+a batched query is shown below from the Marten tests:
+
+snippet: sample_using_query_plan_in_batch_query
+
+
+
+
+
+

src/DocumentDbTests/Reading/query_plans.cs

@@ -0,0 +1,141 @@
+using System.Collections.Generic;
+using System.Drawing;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Marten;
+using Marten.Services.BatchQuerying;
+using Marten.Testing.Documents;
+using Marten.Testing.Harness;
+using Shouldly;
+using Xunit;
+
+namespace DocumentDbTests.Reading;
+
+public class query_plans : IntegrationContext
+{
+    public query_plans(DefaultStoreFixture fixture) : base(fixture)
+    {
+    }
+
+    [Fact]
+    public async Task query_by_query_plan()
+    {
+        await theStore.Advanced.Clean.DeleteDocumentsByTypeAsync(typeof(Target));
+
+        var targets = Target.GenerateRandomData(1000).ToArray();
+        await theStore.BulkInsertDocumentsAsync(targets);
+
+        var blues = await theSession.QueryByPlanAsync(new ColorTargets(Colors.Blue));
+
+        blues.ShouldNotBeEmpty();
+
+        var expected = targets.Where(x => x.Color == Colors.Blue).OrderBy(x => x.Number);
+
+        blues.Select(x => x.Id).ShouldBe(expected.Select(x => x.Id));
+    }
+
+    #region sample_using_query_plan_in_batch_query
+
+    [Fact]
+    public async Task use_as_batch()
+    {
+        await theStore.Advanced.Clean.DeleteDocumentsByTypeAsync(typeof(Target));
+
+        var targets = Target.GenerateRandomData(1000).ToArray();
+        await theStore.BulkInsertDocumentsAsync(targets);
+
+        // Start a batch query
+        var batch = theSession.CreateBatchQuery();
+
+        // Using the ColorTargets plan twice, once for "Blue" and once for "Green" target documents
+        var blueFetcher = batch.QueryByPlan(new ColorTargets(Colors.Blue));
+        var greenFetcher = batch.QueryByPlan(new ColorTargets(Colors.Green));
+
+        // Execute the batch query
+        await batch.Execute();
+
+        // The batched querying in Marten is essentially registering a "future"
+        // for each query, so we'll await each task from above to get at the actual
+        // data returned from batch.Execute() above
+        var blues = await blueFetcher;
+        var greens = await greenFetcher;
+
+        // And the assertion part of our arrange, act, assertion test
+        blues.ShouldNotBeEmpty();
+        greens.ShouldNotBeEmpty();
+
+        var expectedBlues = targets.Where(x => x.Color == Colors.Blue).OrderBy(x => x.Number);
+        var expectedReds = targets.Where(x => x.Color == Colors.Green).OrderBy(x => x.Number);
+
+        blues.Select(x => x.Id).ShouldBe(expectedBlues.Select(x => x.Id));
+        greens.Select(x => x.Id).ShouldBe(expectedReds.Select(x => x.Id));
+    }
+
+    #endregion
+}
+
+#region sample_color_targets
+
+public class ColorTargets: QueryListPlan<Target>
+{
+    public Colors Color { get; }
+
+    public ColorTargets(Colors color)
+    {
+        Color = color;
+    }
+
+    // All we're doing here is just turning around and querying against the session
+    // All the same though, this approach lets you do much more runtime logic
+    // than a compiled query can
+    public override IQueryable<Target> Query(IQuerySession session)
+    {
+        return session.Query<Target>().Where(x => x.Color == Color).OrderBy(x => x.Number);
+    }
+}
+
+// The above is short hand for:
+
+public class LonghandColorTargets: IQueryPlan<IReadOnlyList<Target>>, IBatchQueryPlan<IReadOnlyList<Target>>
+{
+    public Colors Color { get; }
+
+    public LonghandColorTargets(Colors color)
+    {
+        Color = color;
+    }
+
+    public Task<IReadOnlyList<Target>> Fetch(IQuerySession session, CancellationToken token)
+    {
+        return session
+            .Query<Target>()
+            .Where(x => x.Color == Color)
+            .OrderBy(x => x.Number)
+            .ToListAsync(token: token);
+    }
+
+    public Task<IReadOnlyList<Target>> Fetch(IBatchedQuery batch)
+    {
+        return batch
+            .Query<Target>()
+            .Where(x => x.Color == Color)
+            .OrderBy(x => x.Number)
+            .ToList();
+    }
+}
+
+#endregion
+
+public static class query_plan_samples
+{
+    #region sample_using_query_plan
+
+    public static async Task use_query_plan(IQuerySession session, CancellationToken token)
+    {
+        var targets = await session
+            .QueryByPlanAsync(new ColorTargets(Colors.Blue), token);
+    }
+
+    #endregion
+}

src/Marten/IQueryPlan.cs

@@ -0,0 +1,67 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Marten.Linq;
+using Marten.Services.BatchQuerying;
+
+namespace Marten;
+
+/// <summary>
+/// Marten's concept of the "Specification" pattern for reusable
+/// queries. Use this for operations that cannot be supported by Marten compiled queries
+/// </summary>
+/// <typeparam name="T"></typeparam>
+public interface IQueryPlan<T>
+{
+    Task<T> Fetch(IQuerySession session, CancellationToken token);
+}
+
+#region sample_IBatchQueryPlan
+
+/// <summary>
+/// Marten's concept of the "Specification" pattern for reusable
+/// queries within Marten batched queries. Use this for operations that cannot be supported by Marten compiled queries
+/// </summary>
+/// <typeparam name="T"></typeparam>
+public interface IBatchQueryPlan<T>
+{
+    Task<T> Fetch(IBatchedQuery query);
+}
+
+#endregion
+
+/// <summary>
+/// Base class for query plans for a list of items. Implementations of this abstract type
+/// can be used both individually with IQuerySession.QueryByPlan() and with IBatchedQuery.QueryByPlan()
+/// </summary>
+/// <typeparam name="T"></typeparam>
+public abstract class QueryListPlan<T> : IQueryPlan<IReadOnlyList<T>>, IBatchQueryPlan<IReadOnlyList<T>>
+{
+    /// <summary>
+    /// Return an IQueryable<T> from the IQuerySession to define the query plan
+    /// for Marten
+    /// </summary>
+    /// <param name="session"></param>
+    /// <returns></returns>
+    public abstract IQueryable<T> Query(IQuerySession session);
+
+
+    Task<IReadOnlyList<T>> IQueryPlan<IReadOnlyList<T>>.Fetch(IQuerySession session, CancellationToken token)
+    {
+        return Query(session).ToListAsync(token);
+    }
+
+
+    Task<IReadOnlyList<T>> IBatchQueryPlan<IReadOnlyList<T>>.Fetch(IBatchedQuery query)
+    {
+        var queryable = Query(query.Parent) as MartenLinqQueryable<T>;
+        if (queryable == null)
+            throw new InvalidOperationException("Marten is not able to use this QueryListPlan in batch querying");
+
+        var handler = queryable.BuilderListHandler();
+
+        return query.AddItem(handler);
+    }
+}

src/Marten/IQuerySession.cs

@@ -736,4 +736,13 @@ Task<DocumentMetadata> MetadataForAsync<T>(T entity,
     ///     beyond what you can do with LINQ.
     /// </summary>
     IAdvancedSql AdvancedSql { get; }
+
+    /// <summary>
+    /// Use a query plan to execute a query
+    /// </summary>
+    /// <param name="plan"></param>
+    /// <param name="token"></param>
+    /// <typeparam name="T"></typeparam>
+    /// <returns></returns>
+    Task<T> QueryByPlanAsync<T>(IQueryPlan<T> plan, CancellationToken token = default);
 }

src/Marten/Internal/Sessions/QuerySession.cs

@@ -1,6 +1,8 @@
 #nullable enable
 using System;
 using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
 using JasperFx.Core;
 using Marten.Events;
 using Marten.Services;
@@ -111,4 +113,9 @@ public IMartenSessionLogger Logger
     public IDocumentStore DocumentStore => _store;
 
     public IAdvancedSql AdvancedSql => this;
+    public Task<T> QueryByPlanAsync<T>(IQueryPlan<T> plan, CancellationToken token = default)
+    {
+        // This is literally like this *just* to make mocking easier -- even though I don't agree with that often!
+        return plan.Fetch(this, token);
+    }
 }

src/Marten/Linq/MartenLinqQueryable.cs

@@ -138,6 +138,12 @@ IEnumerator IEnumerable.GetEnumerator()
     public IQueryProvider Provider { get; }
     public Expression Expression { get; }
 
+    internal IQueryHandler<IReadOnlyList<T>> BuilderListHandler()
+    {
+        var builder = new LinqQueryParser(MartenProvider, Session, Expression);
+        return builder.BuildListHandler<T>();
+    }
+
     public async Task<IReadOnlyList<TResult>> ToListAsync<TResult>(CancellationToken token)
     {
         var builder = new LinqQueryParser(MartenProvider, Session, Expression);