fix(process worker): fix the crashing of pod (#1119)

* terminate processworker only when running out of memory. * handle SocketException and IOException explicitly in external system calls. * update framework version * update process worker test --------- Co-authored-by: Norbert Truchsess <[email protected]>
eclipse-tractusx · Oct 29, 2024 · a6936a5 · a6936a5
1 parent 9c677fe
commit a6936a5
Show file tree

Hide file tree

Showing 20 changed files with 32 additions and 22 deletions.
diff --git a/src/framework/Framework.Async/Directory.Build.props b/src/framework/Framework.Async/Directory.Build.props
@@ -19,7 +19,7 @@
 
 <Project>
   <PropertyGroup>
-    <VersionPrefix>2.12.0</VersionPrefix>
+    <VersionPrefix>2.13.0</VersionPrefix>
     <VersionSuffix></VersionSuffix>
   </PropertyGroup>
 </Project>
diff --git a/src/framework/Framework.Cors/Directory.Build.props b/src/framework/Framework.Cors/Directory.Build.props
@@ -19,7 +19,7 @@
 
 <Project>
   <PropertyGroup>
-    <VersionPrefix>2.12.0</VersionPrefix>
+    <VersionPrefix>2.13.0</VersionPrefix>
     <VersionSuffix></VersionSuffix>
   </PropertyGroup>
 </Project>
diff --git a/src/framework/Framework.DBAccess/Directory.Build.props b/src/framework/Framework.DBAccess/Directory.Build.props
@@ -19,7 +19,7 @@
 
 <Project>
   <PropertyGroup>
-    <VersionPrefix>2.12.0</VersionPrefix>
+    <VersionPrefix>2.13.0</VersionPrefix>
     <VersionSuffix></VersionSuffix>
   </PropertyGroup>
 </Project>
diff --git a/src/framework/Framework.DateTimeProvider/Directory.Build.props b/src/framework/Framework.DateTimeProvider/Directory.Build.props
@@ -19,7 +19,7 @@
 
 <Project>
   <PropertyGroup>
-    <VersionPrefix>2.12.0</VersionPrefix>
+    <VersionPrefix>2.13.0</VersionPrefix>
     <VersionSuffix></VersionSuffix>
   </PropertyGroup>
 </Project>
diff --git a/src/framework/Framework.DependencyInjection/Directory.Build.props b/src/framework/Framework.DependencyInjection/Directory.Build.props
@@ -19,7 +19,7 @@
 
 <Project>
   <PropertyGroup>
-    <VersionPrefix>2.12.0</VersionPrefix>
+    <VersionPrefix>2.13.0</VersionPrefix>
     <VersionSuffix></VersionSuffix>
   </PropertyGroup>
 </Project>
diff --git a/src/framework/Framework.ErrorHandling.Controller/Directory.Build.props b/src/framework/Framework.ErrorHandling.Controller/Directory.Build.props
@@ -19,7 +19,7 @@
 
 <Project>
   <PropertyGroup>
-    <VersionPrefix>2.12.0</VersionPrefix>
+    <VersionPrefix>2.13.0</VersionPrefix>
     <VersionSuffix></VersionSuffix>
   </PropertyGroup>
 </Project>
diff --git a/src/framework/Framework.ErrorHandling.Web/Directory.Build.props b/src/framework/Framework.ErrorHandling.Web/Directory.Build.props
@@ -19,7 +19,7 @@
 
 <Project>
   <PropertyGroup>
-    <VersionPrefix>2.12.0</VersionPrefix>
+    <VersionPrefix>2.13.0</VersionPrefix>
     <VersionSuffix></VersionSuffix>
   </PropertyGroup>
 </Project>
diff --git a/src/framework/Framework.ErrorHandling/Directory.Build.props b/src/framework/Framework.ErrorHandling/Directory.Build.props
@@ -19,7 +19,7 @@
 
 <Project>
   <PropertyGroup>
-    <VersionPrefix>2.12.0</VersionPrefix>
+    <VersionPrefix>2.13.0</VersionPrefix>
     <VersionSuffix></VersionSuffix>
   </PropertyGroup>
 </Project>
diff --git a/src/framework/Framework.HttpClientExtensions/Directory.Build.props b/src/framework/Framework.HttpClientExtensions/Directory.Build.props
@@ -19,7 +19,7 @@
 
 <Project>
   <PropertyGroup>
-    <VersionPrefix>2.12.0</VersionPrefix>
+    <VersionPrefix>2.13.0</VersionPrefix>
     <VersionSuffix></VersionSuffix>
   </PropertyGroup>
 </Project>
diff --git a/src/framework/Framework.HttpClientExtensions/HttpAsyncResponseMessageExtension.cs b/src/framework/Framework.HttpClientExtensions/HttpAsyncResponseMessageExtension.cs
@@ -18,6 +18,7 @@
  ********************************************************************************/
 
 using Org.Eclipse.TractusX.Portal.Backend.Framework.ErrorHandling;
+using System.Net.Sockets;
 
 namespace Org.Eclipse.TractusX.Portal.Backend.Framework.HttpClientExtensions;
 
@@ -64,6 +65,10 @@ public static async ValueTask<HttpResponseMessage> CatchingIntoServiceExceptionF
                 ? new ServiceException($"call to external system {systemName} failed", e, (recoverOptions & RecoverOptions.REQUEST_EXCEPTION) == RecoverOptions.REQUEST_EXCEPTION)
                 : new ServiceException($"call to external system {systemName} failed with statuscode {(int)e.StatusCode.Value}", e, e.StatusCode.Value, (recoverOptions & RecoverOptions.REQUEST_EXCEPTION) == RecoverOptions.REQUEST_EXCEPTION);
         }
+        catch (SystemException e) when (e is SocketException or IOException)
+        {
+            throw new ServiceException($"call to external system {systemName} failed due to network io", e, (recoverOptions & RecoverOptions.NETWORK) == RecoverOptions.NETWORK);
+        }
         catch (TaskCanceledException e)
         {
             throw new ServiceException($"call to external system {systemName} failed due to timeout", e, (recoverOptions & RecoverOptions.TIMEOUT) == RecoverOptions.TIMEOUT);
@@ -82,7 +87,8 @@ public enum RecoverOptions
         REQUEST_EXCEPTION = 0b_0000_0010,
         TIMEOUT = 0b_0000_0100,
         OTHER_EXCEPTION = 0b_0000_1000,
-        INFRASTRUCTURE = REQUEST_EXCEPTION | TIMEOUT,
-        ALLWAYS = RESPONSE_RECEIVED | REQUEST_EXCEPTION | TIMEOUT | OTHER_EXCEPTION
+        NETWORK = 0b_0001_0000,
+        INFRASTRUCTURE = REQUEST_EXCEPTION | TIMEOUT | NETWORK,
+        ALLWAYS = RESPONSE_RECEIVED | REQUEST_EXCEPTION | TIMEOUT | NETWORK | OTHER_EXCEPTION
     }
 }
diff --git a/src/framework/Framework.IO/Directory.Build.props b/src/framework/Framework.IO/Directory.Build.props
@@ -19,7 +19,7 @@
 
 <Project>
   <PropertyGroup>
-    <VersionPrefix>2.12.0</VersionPrefix>
+    <VersionPrefix>2.13.0</VersionPrefix>
     <VersionSuffix></VersionSuffix>
   </PropertyGroup>
 </Project>
diff --git a/src/framework/Framework.Linq/Directory.Build.props b/src/framework/Framework.Linq/Directory.Build.props
@@ -19,7 +19,7 @@
 
 <Project>
   <PropertyGroup>
-    <VersionPrefix>2.12.0</VersionPrefix>
+    <VersionPrefix>2.13.0</VersionPrefix>
     <VersionSuffix></VersionSuffix>
   </PropertyGroup>
 </Project>
diff --git a/src/framework/Framework.Logging/Directory.Build.props b/src/framework/Framework.Logging/Directory.Build.props
@@ -19,7 +19,7 @@
 
 <Project>
   <PropertyGroup>
-    <VersionPrefix>2.12.0</VersionPrefix>
+    <VersionPrefix>2.13.0</VersionPrefix>
     <VersionSuffix></VersionSuffix>
   </PropertyGroup>
 </Project>
diff --git a/src/framework/Framework.Models/Directory.Build.props b/src/framework/Framework.Models/Directory.Build.props
@@ -19,7 +19,7 @@
 
 <Project>
   <PropertyGroup>
-    <VersionPrefix>2.12.0</VersionPrefix>
+    <VersionPrefix>2.13.0</VersionPrefix>
     <VersionSuffix></VersionSuffix>
   </PropertyGroup>
 </Project>
diff --git a/src/framework/Framework.Seeding/Directory.Build.props b/src/framework/Framework.Seeding/Directory.Build.props
@@ -19,7 +19,7 @@
 
 <Project>
   <PropertyGroup>
-    <VersionPrefix>2.12.0</VersionPrefix>
+    <VersionPrefix>2.13.0</VersionPrefix>
     <VersionSuffix></VersionSuffix>
   </PropertyGroup>
 </Project>
diff --git a/src/framework/Framework.Swagger/Directory.Build.props b/src/framework/Framework.Swagger/Directory.Build.props
@@ -19,7 +19,7 @@
 
 <Project>
   <PropertyGroup>
-    <VersionPrefix>2.12.0</VersionPrefix>
+    <VersionPrefix>2.13.0</VersionPrefix>
     <VersionSuffix></VersionSuffix>
   </PropertyGroup>
 </Project>
diff --git a/src/framework/Framework.Token/Directory.Build.props b/src/framework/Framework.Token/Directory.Build.props
@@ -19,7 +19,7 @@
 
 <Project>
   <PropertyGroup>
-    <VersionPrefix>2.12.0</VersionPrefix>
+    <VersionPrefix>2.13.0</VersionPrefix>
     <VersionSuffix></VersionSuffix>
   </PropertyGroup>
 </Project>
diff --git a/src/framework/Framework.Web/Directory.Build.props b/src/framework/Framework.Web/Directory.Build.props
@@ -19,7 +19,7 @@
 
 <Project>
   <PropertyGroup>
-    <VersionPrefix>2.12.0</VersionPrefix>
+    <VersionPrefix>2.13.0</VersionPrefix>
     <VersionSuffix></VersionSuffix>
   </PropertyGroup>
 </Project>
diff --git a/src/processes/Processes.Worker.Library/ProcessExecutionService.cs b/src/processes/Processes.Worker.Library/ProcessExecutionService.cs
@@ -106,6 +106,11 @@ public async Task ExecuteAsync(CancellationToken stoppingToken)
                     }
                     _logger.LogInformation("finished processing process {processId}", process.Id);
                 }
+                catch (SystemException ex) when (ex is not OutOfMemoryException)
+                {
+                    _logger.LogCritical(ex, "Critical error : processing process {processId} type {processType}: {message}", process.Id, process.ProcessTypeId, ex.Message);
+                    executorRepositories.Clear();
+                }
                 catch (Exception ex) when (ex is not SystemException)
                 {
                     _logger.LogInformation(ex, "error processing process {processId} type {processType}: {message}", process.Id, process.ProcessTypeId, ex.Message);

diff --git a/tests/processes/Processes.Worker.Library.Tests/ProcessExecutionServiceTests.cs b/tests/processes/Processes.Worker.Library.Tests/ProcessExecutionServiceTests.cs
@@ -507,11 +507,10 @@ public async Task ExecuteAsync_WithSystemException_Exits()
         await _service.ExecuteAsync(CancellationToken.None);
 
         // Assert
-        Environment.ExitCode.Should().Be(1);
-        A.CallTo(() => _mockLogger.Log(LogLevel.Information, A<Exception>.That.IsNull(), A<string>.That.Matches(x => x.StartsWith("start processing process")))).MustHaveHappenedOnceExactly();
+        A.CallTo(() => _mockLogger.Log(LogLevel.Information, A<Exception>.That.IsNull(), A<string>.That.Matches(x => x.StartsWith("start processing process")))).MustHaveHappened(3, Times.Exactly);
         A.CallTo(() => _mockLogger.Log(LogLevel.Information, A<Exception>.That.IsNotNull(), A<string>._)).MustNotHaveHappened();
         A.CallTo(() => _mockLogger.Log(LogLevel.Information, A<Exception>.That.IsNull(), A<string>.That.Matches(x => x.StartsWith("finished processing process")))).MustNotHaveHappened();
-        A.CallTo(() => _mockLogger.Log(LogLevel.Error, A<Exception>.That.Matches(e => e != null && e.Message == error.Message), $"processing failed with following Exception {error.Message}")).MustHaveHappenedOnceExactly();
+        A.CallTo(() => _mockLogger.Log(LogLevel.Critical, A<Exception>.That.Matches(e => e != null && e.Message == error.Message), A<string>.That.StartsWith($"Critical error : processing process"))).MustHaveHappened(3, Times.Exactly);
         A.CallTo(() => _portalRepositories.SaveAsync()).MustNotHaveHappened();
     }
 }