From 432e89112613578273021a630009f4a5a0566ffa Mon Sep 17 00:00:00 2001
From: Eric Lidwa <eric.lidwa@nasa.gov>
Date: Thu, 20 Jun 2024 13:22:00 +0000
Subject: [PATCH] raster code proper handling of Thread exception

---
 packages/geo/GeoIndexedRaster.cpp | 33 ++++++++++++++++++++-----------
 packages/geo/GeoIndexedRaster.h   |  2 +-
 packages/geo/GeoParms.h           |  1 +
 packages/geo/RasterObject.cpp     | 12 +++++++++++
 4 files changed, 36 insertions(+), 12 deletions(-)

diff --git a/packages/geo/GeoIndexedRaster.cpp b/packages/geo/GeoIndexedRaster.cpp
index e281fffce..80ec7819d 100644
--- a/packages/geo/GeoIndexedRaster.cpp
+++ b/packages/geo/GeoIndexedRaster.cpp
@@ -439,9 +439,6 @@ bool GeoIndexedRaster::openGeoIndex(const OGRGeometry* geo)
  *----------------------------------------------------------------------------*/
 void GeoIndexedRaster::sampleRasters(OGRGeometry* geo)
 {
-    /* Create additional reader threads if needed */
-    createThreads();
-
     /* For each raster which is marked to be sampled, give it to the reader thread */
     int signaledReaders = 0;
     int i = 0;
@@ -496,8 +493,13 @@ bool GeoIndexedRaster::sample(OGRGeometry* geo, int64_t gps)
 
     if(findRasters(geo) && filterRasters(gps) && updateCache())
     {
-        sampleRasters(geo);
-        status = true;
+        /* Create additional reader threads if needed */
+        status = createThreads();
+        if(status)
+        {
+            sampleRasters(geo);
+            status = true;
+        }
     }
 
     return status;
@@ -677,21 +679,30 @@ void* GeoIndexedRaster::readingThread(void *param)
 /*----------------------------------------------------------------------------
  * createThreads
  *----------------------------------------------------------------------------*/
-void GeoIndexedRaster::createThreads(void)
+bool GeoIndexedRaster::createThreads(void)
 {
     const int threadsNeeded = cache.length();
     const int threadsNow    = readers.length();
     const int newThreadsCnt = threadsNeeded - threadsNow;
 
     if(threadsNeeded <= threadsNow)
-        return;
+        return true;
 
-    for(int i = 0; i < newThreadsCnt; i++)
+    try
     {
-        Reader* r = new Reader(this);
-        readers.add(r);
+        for(int i = 0; i < newThreadsCnt; i++)
+        {
+            Reader* r = new Reader(this);
+            readers.add(r);
+        }
+    }
+    catch (const RunTimeException &e)
+    {
+        ssError |= SS_RESOURCE_LIMIT_ERROR;
+        mlog(CRITICAL, "Failed to create reader threads, needed: %d, created: %d", newThreadsCnt, readers.length() - threadsNow);
     }
-    assert(readers.length() == threadsNeeded);
+
+    return readers.length() == threadsNeeded;
 }
 
 /*----------------------------------------------------------------------------
diff --git a/packages/geo/GeoIndexedRaster.h b/packages/geo/GeoIndexedRaster.h
index 26506b443..90f9a9a64 100644
--- a/packages/geo/GeoIndexedRaster.h
+++ b/packages/geo/GeoIndexedRaster.h
@@ -172,7 +172,7 @@ class GeoIndexedRaster: public RasterObject
 
         static void*    readingThread   (void *param);
 
-        void            createThreads   (void);
+        bool            createThreads   (void);
         bool            updateCache     (void);
         bool            filterRasters   (int64_t gps);
 };
diff --git a/packages/geo/GeoParms.h b/packages/geo/GeoParms.h
index 072e71f12..9350b5c89 100644
--- a/packages/geo/GeoParms.h
+++ b/packages/geo/GeoParms.h
@@ -58,6 +58,7 @@
 #define SS_WRITE_ERROR            (1 << 4)
 #define SS_SUBRASTER_ERROR        (1 << 5)
 #define SS_INDEX_FILE_ERROR       (1 << 6)
+#define SS_RESOURCE_LIMIT_ERROR   (1 << 7)
 
 
 /******************************************************************************
diff --git a/packages/geo/RasterObject.cpp b/packages/geo/RasterObject.cpp
index 7526bedaa..0c3d0a656 100644
--- a/packages/geo/RasterObject.cpp
+++ b/packages/geo/RasterObject.cpp
@@ -263,6 +263,12 @@ int RasterObject::luaSamples(lua_State *L)
             mlog(CRITICAL, "Too many rasters to sample, max allowed: %d, limit your AOI/temporal range or use filters", GeoIndexedRaster::MAX_READER_THREADS);
         }
 
+        if(err & SS_RESOURCE_LIMIT_ERROR)
+        {
+            listvalid = false;
+            mlog(CRITICAL, "System resource limit reached, could not sample rasters");
+        }
+
         /* Create return table */
         lua_createtable(L, slist.length(), 0);
         num_ret++;
@@ -394,6 +400,12 @@ int RasterObject::slist2table(const List<RasterSubset*>& slist, uint32_t errors,
         mlog(CRITICAL, "Some rasters could not be subset, requested memory size > max allowed: %ld MB", RasterSubset::MAX_SIZE / (1024 * 1024));
     }
 
+    if(errors & SS_RESOURCE_LIMIT_ERROR)
+    {
+        listvalid = false;
+        mlog(CRITICAL, "System resource limit reached, could not subset rasters");
+    }
+
     /* Create return table */
     lua_createtable(L, slist.length(), 0);
     num_ret++;