diff --git a/src/Marten.AspNetCore/Daemon/AsyncDaemonHealthCheckExtensions.cs b/src/Marten.AspNetCore/Daemon/AsyncDaemonHealthCheckExtensions.cs index 803f7f5013..e433047262 100644 --- a/src/Marten.AspNetCore/Daemon/AsyncDaemonHealthCheckExtensions.cs +++ b/src/Marten.AspNetCore/Daemon/AsyncDaemonHealthCheckExtensions.cs @@ -1,4 +1,3 @@ -#nullable enable using System; using System.Collections.Concurrent; using System.Collections.Generic; @@ -17,20 +16,32 @@ namespace Marten.Events.Daemon; public static class AsyncDaemonHealthCheckExtensions { /// - /// Adds a health check for Martens Async Daemon. - /// The health check will verify that no async projection progression is lagging behind more than the - /// The check will return if any progression is more than behind the highWaterMark OR if any exception is thrown while doing the check. - /// - /// + /// Adds a health check for Martens Async Daemon. + /// The health check will verify that no async projection progression is lagging behind more than the + /// + /// The check will return if any progression is more than + /// behind the highWaterMark OR if any exception is thrown while doing the check. + /// + /// /// Customized Injection Example: services.AddHealthChecks().AddAsyncDaemonHealthCheck(150); /// - /// - /// Also - remember to add app.MapHealthChecks("/your-health-path") to the middleware pipeline + /// + /// Also - remember to add app.MapHealthChecks("/your-health-path") to the middleware pipeline /// - /// - /// (optional) Acceptable lag of an eventprojection before it's considered unhealthy - defaults to 100 - /// (optional) Treat projection as healthy if maxEventLag exceeded, but projection sequence changed since last check in given time - defaults to null (uses just maxEventLag) - /// If healthy: - else + /// + /// + /// + /// + /// (optional) Acceptable lag of an eventprojection before it's considered unhealthy - defaults + /// to 100 + /// + /// + /// (optional) Treat projection as healthy if maxEventLag exceeded, but projection sequence + /// changed since last check in given time - defaults to null (uses just maxEventLag) + /// + /// + /// If healthy: - else + /// public static IHealthChecksBuilder AddMartenAsyncDaemonHealthCheck( this IHealthChecksBuilder builder, int maxEventLag = 100, @@ -46,39 +57,39 @@ public static IHealthChecksBuilder AddMartenAsyncDaemonHealthCheck( } /// - /// Internal class used to DI settings to async daemon health check + /// Internal class used to DI settings to async daemon health check /// /// /// internal record AsyncDaemonHealthCheckSettings(int MaxEventLag, TimeSpan? MaxSameLagTime = null); /// - /// Health check implementation + /// Health check implementation /// internal class AsyncDaemonHealthCheck: IHealthCheck { - /// - /// The to check health for. - /// - private readonly IDocumentStore _store; + private readonly ConcurrentDictionary + _lastProjectionsChecks = new(); /// - /// The allowed event projection processing lag compared to the HighWaterMark. + /// The allowed event projection processing lag compared to the HighWaterMark. /// private readonly int _maxEventLag; /// - /// The allowed time for event projection is lagging (by maxEventLag). - /// If not provided every projection is considered lagging if HighWaterMark - projection.Position >= maxEventLag. - /// If provided only if projection.Position is still the same for given time. - /// When you want to rely only on time just set _maxEventLag=1 and maxSameLagTime to desired value. + /// The allowed time for event projection is lagging (by maxEventLag). + /// If not provided every projection is considered lagging if HighWaterMark - projection.Position >= maxEventLag. + /// If provided only if projection.Position is still the same for given time. + /// When you want to rely only on time just set _maxEventLag=1 and maxSameLagTime to desired value. /// private readonly TimeSpan? _maxSameLagTime; - private readonly TimeProvider _timeProvider; + /// + /// The to check health for. + /// + private readonly IDocumentStore _store; - private readonly ConcurrentDictionary - _lastProjectionsChecks = new(); + private readonly TimeProvider _timeProvider; public AsyncDaemonHealthCheck(IDocumentStore store, AsyncDaemonHealthCheckSettings settings, TimeProvider timeProvider) @@ -89,7 +100,7 @@ public AsyncDaemonHealthCheck(IDocumentStore store, AsyncDaemonHealthCheckSettin _maxSameLagTime = settings.MaxSameLagTime; } - public Task CheckHealthAsync( + public async Task CheckHealthAsync( HealthCheckContext context, CancellationToken cancellationToken = default ) @@ -101,26 +112,47 @@ public Task CheckHealthAsync( .Select(x => $"{x.ProjectionName}:All") .ToHashSet(); - var database = _store.Storage.Database; + var databases = await _store.Storage.AllDatabases().ConfigureAwait(false); + + if (databases.Count == 1) + { + return await CheckProjectionHealthAsync(databases[0], projectionsToCheck, cancellationToken) + .ConfigureAwait(false); + } + else + { + // Cheesy, but just returning unhealthy + foreach (var database in databases) + { + var healthCheck = + await CheckProjectionHealthAsync(database, projectionsToCheck, cancellationToken) + .ConfigureAwait(false); + if (healthCheck.Status != HealthStatus.Healthy) + { + return healthCheck; + } + } + } - return CheckProjectionHealthAsync(database, projectionsToCheck, cancellationToken); + return HealthCheckResult.Healthy("Healthy"); } catch (Exception ex) { - return Task.FromResult(HealthCheckResult.Unhealthy($"Unhealthy: {ex.Message}", ex)); + return HealthCheckResult.Unhealthy($"Unhealthy: {ex.Message}", ex); } } - public async Task CheckProjectionHealthAsync(IMartenDatabase database, HashSet projectionsToCheck, + public async Task CheckProjectionHealthAsync(IMartenDatabase database, + HashSet projectionsToCheck, CancellationToken cancellationToken) { - var allProgress = await database.AllProjectionProgress(token: cancellationToken) + var allProgress = await database.AllProjectionProgress(cancellationToken) .ConfigureAwait(true); var highWaterMark = allProgress.FirstOrDefault(x => string.Equals("HighWaterMark", x.ShardName)); if (highWaterMark is null) { - return Microsoft.Extensions.Diagnostics.HealthChecks.HealthCheckResult.Healthy("Healthy"); + return HealthCheckResult.Healthy("Healthy"); } var projectionMarks = allProgress.Where(x => !string.Equals("HighWaterMark", x.ShardName)).ToArray(); @@ -136,10 +168,10 @@ public async Task CheckProjectionHealthAsync(IMartenDatabase if (_maxSameLagTime is null) { return laggingProjections.Any() - ? Microsoft.Extensions.Diagnostics.HealthChecks.HealthCheckResult.Unhealthy( + ? HealthCheckResult.Unhealthy( $"Unhealthy: Async projection sequence is more than {_maxEventLag} events behind for projection(s): {laggingProjections.Select(x => x.ShardName).Join(", ")}" ) - : Microsoft.Extensions.Diagnostics.HealthChecks.HealthCheckResult.Healthy("Healthy"); + : HealthCheckResult.Healthy("Healthy"); } var now = _timeProvider.GetUtcNow().UtcDateTime; @@ -169,10 +201,10 @@ public async Task CheckProjectionHealthAsync(IMartenDatabase } return projectionsLaggingWithSamePositionForGivenTime.Any() - ? Microsoft.Extensions.Diagnostics.HealthChecks.HealthCheckResult.Unhealthy( + ? HealthCheckResult.Unhealthy( $"Unhealthy: Async projection sequence is more than {_maxEventLag} events behind with same sequence for more than {_maxSameLagTime} for projection(s): {projectionsLaggingWithSamePositionForGivenTime.Select(x => x.ShardName).Join(", ")}" ) - : Microsoft.Extensions.Diagnostics.HealthChecks.HealthCheckResult.Healthy("Healthy"); + : HealthCheckResult.Healthy("Healthy"); } } }