From a6936a5a2a927ddda1f86fc2a5ced29ad12f8b0e Mon Sep 17 00:00:00 2001 From: "Dhirender Singh (Cofinity-X)" <144212607+dhiren-singh-007@users.noreply.github.com> Date: Tue, 29 Oct 2024 13:16:07 +0100 Subject: [PATCH] fix(process worker): fix the crashing of pod (#1119) * terminate processworker only when running out of memory. * handle SocketException and IOException explicitly in external system calls. * update framework version * update process worker test --------- Co-authored-by: Norbert Truchsess --- src/framework/Framework.Async/Directory.Build.props | 2 +- src/framework/Framework.Cors/Directory.Build.props | 2 +- src/framework/Framework.DBAccess/Directory.Build.props | 2 +- .../Framework.DateTimeProvider/Directory.Build.props | 2 +- .../Directory.Build.props | 2 +- .../Directory.Build.props | 2 +- .../Framework.ErrorHandling.Web/Directory.Build.props | 2 +- .../Framework.ErrorHandling/Directory.Build.props | 2 +- .../Directory.Build.props | 2 +- .../HttpAsyncResponseMessageExtension.cs | 10 ++++++++-- src/framework/Framework.IO/Directory.Build.props | 2 +- src/framework/Framework.Linq/Directory.Build.props | 2 +- src/framework/Framework.Logging/Directory.Build.props | 2 +- src/framework/Framework.Models/Directory.Build.props | 2 +- src/framework/Framework.Seeding/Directory.Build.props | 2 +- src/framework/Framework.Swagger/Directory.Build.props | 2 +- src/framework/Framework.Token/Directory.Build.props | 2 +- src/framework/Framework.Web/Directory.Build.props | 2 +- .../ProcessExecutionService.cs | 5 +++++ .../ProcessExecutionServiceTests.cs | 5 ++--- 20 files changed, 32 insertions(+), 22 deletions(-) diff --git a/src/framework/Framework.Async/Directory.Build.props b/src/framework/Framework.Async/Directory.Build.props index 8fa21cbd24..6e30a7b3ec 100644 --- a/src/framework/Framework.Async/Directory.Build.props +++ b/src/framework/Framework.Async/Directory.Build.props @@ -19,7 +19,7 @@ - 2.12.0 + 2.13.0 diff --git a/src/framework/Framework.Cors/Directory.Build.props b/src/framework/Framework.Cors/Directory.Build.props index 8fa21cbd24..6e30a7b3ec 100644 --- a/src/framework/Framework.Cors/Directory.Build.props +++ b/src/framework/Framework.Cors/Directory.Build.props @@ -19,7 +19,7 @@ - 2.12.0 + 2.13.0 diff --git a/src/framework/Framework.DBAccess/Directory.Build.props b/src/framework/Framework.DBAccess/Directory.Build.props index 8fa21cbd24..6e30a7b3ec 100644 --- a/src/framework/Framework.DBAccess/Directory.Build.props +++ b/src/framework/Framework.DBAccess/Directory.Build.props @@ -19,7 +19,7 @@ - 2.12.0 + 2.13.0 diff --git a/src/framework/Framework.DateTimeProvider/Directory.Build.props b/src/framework/Framework.DateTimeProvider/Directory.Build.props index 8fa21cbd24..6e30a7b3ec 100644 --- a/src/framework/Framework.DateTimeProvider/Directory.Build.props +++ b/src/framework/Framework.DateTimeProvider/Directory.Build.props @@ -19,7 +19,7 @@ - 2.12.0 + 2.13.0 diff --git a/src/framework/Framework.DependencyInjection/Directory.Build.props b/src/framework/Framework.DependencyInjection/Directory.Build.props index 8fa21cbd24..6e30a7b3ec 100644 --- a/src/framework/Framework.DependencyInjection/Directory.Build.props +++ b/src/framework/Framework.DependencyInjection/Directory.Build.props @@ -19,7 +19,7 @@ - 2.12.0 + 2.13.0 diff --git a/src/framework/Framework.ErrorHandling.Controller/Directory.Build.props b/src/framework/Framework.ErrorHandling.Controller/Directory.Build.props index 8fa21cbd24..6e30a7b3ec 100644 --- a/src/framework/Framework.ErrorHandling.Controller/Directory.Build.props +++ b/src/framework/Framework.ErrorHandling.Controller/Directory.Build.props @@ -19,7 +19,7 @@ - 2.12.0 + 2.13.0 diff --git a/src/framework/Framework.ErrorHandling.Web/Directory.Build.props b/src/framework/Framework.ErrorHandling.Web/Directory.Build.props index 8fa21cbd24..6e30a7b3ec 100644 --- a/src/framework/Framework.ErrorHandling.Web/Directory.Build.props +++ b/src/framework/Framework.ErrorHandling.Web/Directory.Build.props @@ -19,7 +19,7 @@ - 2.12.0 + 2.13.0 diff --git a/src/framework/Framework.ErrorHandling/Directory.Build.props b/src/framework/Framework.ErrorHandling/Directory.Build.props index 8fa21cbd24..6e30a7b3ec 100644 --- a/src/framework/Framework.ErrorHandling/Directory.Build.props +++ b/src/framework/Framework.ErrorHandling/Directory.Build.props @@ -19,7 +19,7 @@ - 2.12.0 + 2.13.0 diff --git a/src/framework/Framework.HttpClientExtensions/Directory.Build.props b/src/framework/Framework.HttpClientExtensions/Directory.Build.props index 8fa21cbd24..6e30a7b3ec 100644 --- a/src/framework/Framework.HttpClientExtensions/Directory.Build.props +++ b/src/framework/Framework.HttpClientExtensions/Directory.Build.props @@ -19,7 +19,7 @@ - 2.12.0 + 2.13.0 diff --git a/src/framework/Framework.HttpClientExtensions/HttpAsyncResponseMessageExtension.cs b/src/framework/Framework.HttpClientExtensions/HttpAsyncResponseMessageExtension.cs index 1a841ab0de..f4110e507b 100644 --- a/src/framework/Framework.HttpClientExtensions/HttpAsyncResponseMessageExtension.cs +++ b/src/framework/Framework.HttpClientExtensions/HttpAsyncResponseMessageExtension.cs @@ -18,6 +18,7 @@ ********************************************************************************/ using Org.Eclipse.TractusX.Portal.Backend.Framework.ErrorHandling; +using System.Net.Sockets; namespace Org.Eclipse.TractusX.Portal.Backend.Framework.HttpClientExtensions; @@ -64,6 +65,10 @@ public static async ValueTask CatchingIntoServiceExceptionF ? new ServiceException($"call to external system {systemName} failed", e, (recoverOptions & RecoverOptions.REQUEST_EXCEPTION) == RecoverOptions.REQUEST_EXCEPTION) : new ServiceException($"call to external system {systemName} failed with statuscode {(int)e.StatusCode.Value}", e, e.StatusCode.Value, (recoverOptions & RecoverOptions.REQUEST_EXCEPTION) == RecoverOptions.REQUEST_EXCEPTION); } + catch (SystemException e) when (e is SocketException or IOException) + { + throw new ServiceException($"call to external system {systemName} failed due to network io", e, (recoverOptions & RecoverOptions.NETWORK) == RecoverOptions.NETWORK); + } catch (TaskCanceledException e) { throw new ServiceException($"call to external system {systemName} failed due to timeout", e, (recoverOptions & RecoverOptions.TIMEOUT) == RecoverOptions.TIMEOUT); @@ -82,7 +87,8 @@ public enum RecoverOptions REQUEST_EXCEPTION = 0b_0000_0010, TIMEOUT = 0b_0000_0100, OTHER_EXCEPTION = 0b_0000_1000, - INFRASTRUCTURE = REQUEST_EXCEPTION | TIMEOUT, - ALLWAYS = RESPONSE_RECEIVED | REQUEST_EXCEPTION | TIMEOUT | OTHER_EXCEPTION + NETWORK = 0b_0001_0000, + INFRASTRUCTURE = REQUEST_EXCEPTION | TIMEOUT | NETWORK, + ALLWAYS = RESPONSE_RECEIVED | REQUEST_EXCEPTION | TIMEOUT | NETWORK | OTHER_EXCEPTION } } diff --git a/src/framework/Framework.IO/Directory.Build.props b/src/framework/Framework.IO/Directory.Build.props index 8fa21cbd24..6e30a7b3ec 100644 --- a/src/framework/Framework.IO/Directory.Build.props +++ b/src/framework/Framework.IO/Directory.Build.props @@ -19,7 +19,7 @@ - 2.12.0 + 2.13.0 diff --git a/src/framework/Framework.Linq/Directory.Build.props b/src/framework/Framework.Linq/Directory.Build.props index 8fa21cbd24..6e30a7b3ec 100644 --- a/src/framework/Framework.Linq/Directory.Build.props +++ b/src/framework/Framework.Linq/Directory.Build.props @@ -19,7 +19,7 @@ - 2.12.0 + 2.13.0 diff --git a/src/framework/Framework.Logging/Directory.Build.props b/src/framework/Framework.Logging/Directory.Build.props index 8fa21cbd24..6e30a7b3ec 100644 --- a/src/framework/Framework.Logging/Directory.Build.props +++ b/src/framework/Framework.Logging/Directory.Build.props @@ -19,7 +19,7 @@ - 2.12.0 + 2.13.0 diff --git a/src/framework/Framework.Models/Directory.Build.props b/src/framework/Framework.Models/Directory.Build.props index 8fa21cbd24..6e30a7b3ec 100644 --- a/src/framework/Framework.Models/Directory.Build.props +++ b/src/framework/Framework.Models/Directory.Build.props @@ -19,7 +19,7 @@ - 2.12.0 + 2.13.0 diff --git a/src/framework/Framework.Seeding/Directory.Build.props b/src/framework/Framework.Seeding/Directory.Build.props index 8fa21cbd24..6e30a7b3ec 100644 --- a/src/framework/Framework.Seeding/Directory.Build.props +++ b/src/framework/Framework.Seeding/Directory.Build.props @@ -19,7 +19,7 @@ - 2.12.0 + 2.13.0 diff --git a/src/framework/Framework.Swagger/Directory.Build.props b/src/framework/Framework.Swagger/Directory.Build.props index 8fa21cbd24..6e30a7b3ec 100644 --- a/src/framework/Framework.Swagger/Directory.Build.props +++ b/src/framework/Framework.Swagger/Directory.Build.props @@ -19,7 +19,7 @@ - 2.12.0 + 2.13.0 diff --git a/src/framework/Framework.Token/Directory.Build.props b/src/framework/Framework.Token/Directory.Build.props index 8fa21cbd24..6e30a7b3ec 100644 --- a/src/framework/Framework.Token/Directory.Build.props +++ b/src/framework/Framework.Token/Directory.Build.props @@ -19,7 +19,7 @@ - 2.12.0 + 2.13.0 diff --git a/src/framework/Framework.Web/Directory.Build.props b/src/framework/Framework.Web/Directory.Build.props index 8fa21cbd24..6e30a7b3ec 100644 --- a/src/framework/Framework.Web/Directory.Build.props +++ b/src/framework/Framework.Web/Directory.Build.props @@ -19,7 +19,7 @@ - 2.12.0 + 2.13.0 diff --git a/src/processes/Processes.Worker.Library/ProcessExecutionService.cs b/src/processes/Processes.Worker.Library/ProcessExecutionService.cs index 03aae8ac07..d2a00a32d0 100644 --- a/src/processes/Processes.Worker.Library/ProcessExecutionService.cs +++ b/src/processes/Processes.Worker.Library/ProcessExecutionService.cs @@ -106,6 +106,11 @@ public async Task ExecuteAsync(CancellationToken stoppingToken) } _logger.LogInformation("finished processing process {processId}", process.Id); } + catch (SystemException ex) when (ex is not OutOfMemoryException) + { + _logger.LogCritical(ex, "Critical error : processing process {processId} type {processType}: {message}", process.Id, process.ProcessTypeId, ex.Message); + executorRepositories.Clear(); + } catch (Exception ex) when (ex is not SystemException) { _logger.LogInformation(ex, "error processing process {processId} type {processType}: {message}", process.Id, process.ProcessTypeId, ex.Message); diff --git a/tests/processes/Processes.Worker.Library.Tests/ProcessExecutionServiceTests.cs b/tests/processes/Processes.Worker.Library.Tests/ProcessExecutionServiceTests.cs index b9a8d18465..38a9b33a5d 100644 --- a/tests/processes/Processes.Worker.Library.Tests/ProcessExecutionServiceTests.cs +++ b/tests/processes/Processes.Worker.Library.Tests/ProcessExecutionServiceTests.cs @@ -507,11 +507,10 @@ public async Task ExecuteAsync_WithSystemException_Exits() await _service.ExecuteAsync(CancellationToken.None); // Assert - Environment.ExitCode.Should().Be(1); - A.CallTo(() => _mockLogger.Log(LogLevel.Information, A.That.IsNull(), A.That.Matches(x => x.StartsWith("start processing process")))).MustHaveHappenedOnceExactly(); + A.CallTo(() => _mockLogger.Log(LogLevel.Information, A.That.IsNull(), A.That.Matches(x => x.StartsWith("start processing process")))).MustHaveHappened(3, Times.Exactly); A.CallTo(() => _mockLogger.Log(LogLevel.Information, A.That.IsNotNull(), A._)).MustNotHaveHappened(); A.CallTo(() => _mockLogger.Log(LogLevel.Information, A.That.IsNull(), A.That.Matches(x => x.StartsWith("finished processing process")))).MustNotHaveHappened(); - A.CallTo(() => _mockLogger.Log(LogLevel.Error, A.That.Matches(e => e != null && e.Message == error.Message), $"processing failed with following Exception {error.Message}")).MustHaveHappenedOnceExactly(); + A.CallTo(() => _mockLogger.Log(LogLevel.Critical, A.That.Matches(e => e != null && e.Message == error.Message), A.That.StartsWith($"Critical error : processing process"))).MustHaveHappened(3, Times.Exactly); A.CallTo(() => _portalRepositories.SaveAsync()).MustNotHaveHappened(); } }