diff --git a/src/Marten.AspNetCore/Daemon/AsyncDaemonHealthCheckExtensions.cs b/src/Marten.AspNetCore/Daemon/AsyncDaemonHealthCheckExtensions.cs
index 803f7f5013..e433047262 100644
--- a/src/Marten.AspNetCore/Daemon/AsyncDaemonHealthCheckExtensions.cs
+++ b/src/Marten.AspNetCore/Daemon/AsyncDaemonHealthCheckExtensions.cs
@@ -1,4 +1,3 @@
-#nullable enable
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
@@ -17,20 +16,32 @@ namespace Marten.Events.Daemon;
public static class AsyncDaemonHealthCheckExtensions
{
///
- /// Adds a health check for Martens Async Daemon.
- /// The health check will verify that no async projection progression is lagging behind more than the
- /// The check will return if any progression is more than behind the highWaterMark OR if any exception is thrown while doing the check.
- ///
- ///
+ /// Adds a health check for Martens Async Daemon.
+ /// The health check will verify that no async projection progression is lagging behind more than the
+ ///
+ /// The check will return if any progression is more than
+ /// behind the highWaterMark OR if any exception is thrown while doing the check.
+ ///
+ ///
/// Customized Injection Example: services.AddHealthChecks().AddAsyncDaemonHealthCheck(150);
///
- ///
- /// Also - remember to add app.MapHealthChecks("/your-health-path") to the middleware pipeline
+ ///
+ /// Also - remember to add app.MapHealthChecks("/your-health-path") to the middleware pipeline
///
- ///
- /// (optional) Acceptable lag of an eventprojection before it's considered unhealthy - defaults to 100
- /// (optional) Treat projection as healthy if maxEventLag exceeded, but projection sequence changed since last check in given time - defaults to null (uses just maxEventLag)
- /// If healthy: - else
+ ///
+ ///
+ ///
+ ///
+ /// (optional) Acceptable lag of an eventprojection before it's considered unhealthy - defaults
+ /// to 100
+ ///
+ ///
+ /// (optional) Treat projection as healthy if maxEventLag exceeded, but projection sequence
+ /// changed since last check in given time - defaults to null (uses just maxEventLag)
+ ///
+ ///
+ /// If healthy: - else
+ ///
public static IHealthChecksBuilder AddMartenAsyncDaemonHealthCheck(
this IHealthChecksBuilder builder,
int maxEventLag = 100,
@@ -46,39 +57,39 @@ public static IHealthChecksBuilder AddMartenAsyncDaemonHealthCheck(
}
///
- /// Internal class used to DI settings to async daemon health check
+ /// Internal class used to DI settings to async daemon health check
///
///
///
internal record AsyncDaemonHealthCheckSettings(int MaxEventLag, TimeSpan? MaxSameLagTime = null);
///
- /// Health check implementation
+ /// Health check implementation
///
internal class AsyncDaemonHealthCheck: IHealthCheck
{
- ///
- /// The to check health for.
- ///
- private readonly IDocumentStore _store;
+ private readonly ConcurrentDictionary
+ _lastProjectionsChecks = new();
///
- /// The allowed event projection processing lag compared to the HighWaterMark.
+ /// The allowed event projection processing lag compared to the HighWaterMark.
///
private readonly int _maxEventLag;
///
- /// The allowed time for event projection is lagging (by maxEventLag).
- /// If not provided every projection is considered lagging if HighWaterMark - projection.Position >= maxEventLag.
- /// If provided only if projection.Position is still the same for given time.
- /// When you want to rely only on time just set _maxEventLag=1 and maxSameLagTime to desired value.
+ /// The allowed time for event projection is lagging (by maxEventLag).
+ /// If not provided every projection is considered lagging if HighWaterMark - projection.Position >= maxEventLag.
+ /// If provided only if projection.Position is still the same for given time.
+ /// When you want to rely only on time just set _maxEventLag=1 and maxSameLagTime to desired value.
///
private readonly TimeSpan? _maxSameLagTime;
- private readonly TimeProvider _timeProvider;
+ ///
+ /// The to check health for.
+ ///
+ private readonly IDocumentStore _store;
- private readonly ConcurrentDictionary
- _lastProjectionsChecks = new();
+ private readonly TimeProvider _timeProvider;
public AsyncDaemonHealthCheck(IDocumentStore store, AsyncDaemonHealthCheckSettings settings,
TimeProvider timeProvider)
@@ -89,7 +100,7 @@ public AsyncDaemonHealthCheck(IDocumentStore store, AsyncDaemonHealthCheckSettin
_maxSameLagTime = settings.MaxSameLagTime;
}
- public Task CheckHealthAsync(
+ public async Task CheckHealthAsync(
HealthCheckContext context,
CancellationToken cancellationToken = default
)
@@ -101,26 +112,47 @@ public Task CheckHealthAsync(
.Select(x => $"{x.ProjectionName}:All")
.ToHashSet();
- var database = _store.Storage.Database;
+ var databases = await _store.Storage.AllDatabases().ConfigureAwait(false);
+
+ if (databases.Count == 1)
+ {
+ return await CheckProjectionHealthAsync(databases[0], projectionsToCheck, cancellationToken)
+ .ConfigureAwait(false);
+ }
+ else
+ {
+ // Cheesy, but just returning unhealthy
+ foreach (var database in databases)
+ {
+ var healthCheck =
+ await CheckProjectionHealthAsync(database, projectionsToCheck, cancellationToken)
+ .ConfigureAwait(false);
+ if (healthCheck.Status != HealthStatus.Healthy)
+ {
+ return healthCheck;
+ }
+ }
+ }
- return CheckProjectionHealthAsync(database, projectionsToCheck, cancellationToken);
+ return HealthCheckResult.Healthy("Healthy");
}
catch (Exception ex)
{
- return Task.FromResult(HealthCheckResult.Unhealthy($"Unhealthy: {ex.Message}", ex));
+ return HealthCheckResult.Unhealthy($"Unhealthy: {ex.Message}", ex);
}
}
- public async Task CheckProjectionHealthAsync(IMartenDatabase database, HashSet projectionsToCheck,
+ public async Task CheckProjectionHealthAsync(IMartenDatabase database,
+ HashSet projectionsToCheck,
CancellationToken cancellationToken)
{
- var allProgress = await database.AllProjectionProgress(token: cancellationToken)
+ var allProgress = await database.AllProjectionProgress(cancellationToken)
.ConfigureAwait(true);
var highWaterMark = allProgress.FirstOrDefault(x => string.Equals("HighWaterMark", x.ShardName));
if (highWaterMark is null)
{
- return Microsoft.Extensions.Diagnostics.HealthChecks.HealthCheckResult.Healthy("Healthy");
+ return HealthCheckResult.Healthy("Healthy");
}
var projectionMarks = allProgress.Where(x => !string.Equals("HighWaterMark", x.ShardName)).ToArray();
@@ -136,10 +168,10 @@ public async Task CheckProjectionHealthAsync(IMartenDatabase
if (_maxSameLagTime is null)
{
return laggingProjections.Any()
- ? Microsoft.Extensions.Diagnostics.HealthChecks.HealthCheckResult.Unhealthy(
+ ? HealthCheckResult.Unhealthy(
$"Unhealthy: Async projection sequence is more than {_maxEventLag} events behind for projection(s): {laggingProjections.Select(x => x.ShardName).Join(", ")}"
)
- : Microsoft.Extensions.Diagnostics.HealthChecks.HealthCheckResult.Healthy("Healthy");
+ : HealthCheckResult.Healthy("Healthy");
}
var now = _timeProvider.GetUtcNow().UtcDateTime;
@@ -169,10 +201,10 @@ public async Task CheckProjectionHealthAsync(IMartenDatabase
}
return projectionsLaggingWithSamePositionForGivenTime.Any()
- ? Microsoft.Extensions.Diagnostics.HealthChecks.HealthCheckResult.Unhealthy(
+ ? HealthCheckResult.Unhealthy(
$"Unhealthy: Async projection sequence is more than {_maxEventLag} events behind with same sequence for more than {_maxSameLagTime} for projection(s): {projectionsLaggingWithSamePositionForGivenTime.Select(x => x.ShardName).Join(", ")}"
)
- : Microsoft.Extensions.Diagnostics.HealthChecks.HealthCheckResult.Healthy("Healthy");
+ : HealthCheckResult.Healthy("Healthy");
}
}
}