seperated serila and batch code for indexed rasters

SlideRuleEarth · Dec 19, 2024 · bdb593c · bdb593c
1 parent 25697aa
commit bdb593c
Show file tree

Hide file tree

Showing 16 changed files with 1,724 additions and 1,538 deletions.
diff --git a/datasets/bluetopo/package/BlueTopoBathyRaster.cpp b/datasets/bluetopo/package/BlueTopoBathyRaster.cpp
@@ -71,9 +71,18 @@ BlueTopoBathyRaster::~BlueTopoBathyRaster(void) = default;
 /*----------------------------------------------------------------------------
  * getIndexFile
  *----------------------------------------------------------------------------*/
-void BlueTopoBathyRaster::getIndexFile(const OGRGeometry* geo, std::string& file, const std::vector<point_info_t>* points)
+void BlueTopoBathyRaster::getIndexFile(const OGRGeometry* geo, std::string& file)
 {
     static_cast<void>(geo);
+    file = indexFile;
+    mlog(DEBUG, "Using %s", file.c_str());
+}
+
+/*----------------------------------------------------------------------------
+ * getIndexFile
+ *----------------------------------------------------------------------------*/
+void BlueTopoBathyRaster::getIndexFile(const std::vector<point_info_t>* points, std::string& file)
+{
     static_cast<void>(points);
     file = indexFile;
     mlog(DEBUG, "Using %s", file.c_str());

diff --git a/datasets/bluetopo/package/BlueTopoBathyRaster.h b/datasets/bluetopo/package/BlueTopoBathyRaster.h
@@ -73,7 +73,8 @@ class BlueTopoBathyRaster: public GeoIndexedRaster
                 BlueTopoBathyRaster (lua_State* L, RequestFields* rqst_parms, const char* key);
                ~BlueTopoBathyRaster (void) override;
 
-        void    getIndexFile (const OGRGeometry* geo, std::string& file, const std::vector<point_info_t>* points) final;
+        void    getIndexFile (const OGRGeometry* geo, std::string& file) final;
+        void    getIndexFile (const std::vector<point_info_t>* points, std::string& file) final;
         bool    findRasters  (raster_finder_t* finder) final;
 
         /*--------------------------------------------------------------------

diff --git a/datasets/gebco/package/GebcoBathyRaster.cpp b/datasets/gebco/package/GebcoBathyRaster.cpp
@@ -83,14 +83,22 @@ GebcoBathyRaster::~GebcoBathyRaster(void) = default;
 /*----------------------------------------------------------------------------
  * getIndexFile
  *----------------------------------------------------------------------------*/
-void GebcoBathyRaster::getIndexFile(const OGRGeometry* geo, std::string& file, const std::vector<point_info_t>* points)
+void GebcoBathyRaster::getIndexFile(const OGRGeometry* geo, std::string& file)
 {
     static_cast<void>(geo);
-    static_cast<void>(points);
     file = filePath + "/" + indexFile;
     mlog(DEBUG, "Using index file: %s", file.c_str());
 }
 
+/*----------------------------------------------------------------------------
+ * getIndexFile
+ *----------------------------------------------------------------------------*/
+void GebcoBathyRaster::getIndexFile(const std::vector<point_info_t>* points, std::string& file)
+{
+    static_cast<void>(points);
+    file = filePath + "/" + indexFile;
+    mlog(DEBUG, "Using index file: %s", file.c_str());
+}
 
 /*----------------------------------------------------------------------------
  * findRasters

diff --git a/datasets/gebco/package/GebcoBathyRaster.h b/datasets/gebco/package/GebcoBathyRaster.h
@@ -71,7 +71,8 @@ class GebcoBathyRaster: public GeoIndexedRaster
                 GebcoBathyRaster (lua_State* L, RequestFields* rqst_parms, const char* key);
                ~GebcoBathyRaster (void) override;
 
-        void    getIndexFile     (const OGRGeometry* geo, std::string& file, const std::vector<point_info_t>* points) final;
+        void    getIndexFile     (const OGRGeometry* geo, std::string& file) final;
+        void    getIndexFile     (const std::vector<point_info_t>* points, std::string& file) final;
         bool    findRasters      (raster_finder_t* finder) final;
 
         /*--------------------------------------------------------------------

diff --git a/datasets/landsat/package/LandsatHlsRaster.cpp b/datasets/landsat/package/LandsatHlsRaster.cpp
@@ -143,9 +143,18 @@ LandsatHlsRaster::~LandsatHlsRaster(void)
 /*----------------------------------------------------------------------------
  * getIndexFile
  *----------------------------------------------------------------------------*/
-void LandsatHlsRaster::getIndexFile(const OGRGeometry* geo, std::string& file, const std::vector<point_info_t>* points)
+void LandsatHlsRaster::getIndexFile(const OGRGeometry* geo, std::string& file)
 {
     static_cast<void>(geo);
+    file = indexFile;
+    // mlog(DEBUG, "Using %s", file.c_str());
+}
+
+/*----------------------------------------------------------------------------
+ * getIndexFile
+ *----------------------------------------------------------------------------*/
+void LandsatHlsRaster::getIndexFile(const std::vector<point_info_t>* points, std::string& file)
+{
     static_cast<void>(points);
     file = indexFile;
     // mlog(DEBUG, "Using %s", file.c_str());

diff --git a/datasets/landsat/package/LandsatHlsRaster.h b/datasets/landsat/package/LandsatHlsRaster.h
@@ -91,10 +91,11 @@ class LandsatHlsRaster: public GeoIndexedRaster
                  LandsatHlsRaster    (lua_State* L, RequestFields* rqst_parms, const char* key);
                 ~LandsatHlsRaster    (void) override;
 
-        void     getIndexFile        (const OGRGeometry* geo, std::string& file, const std::vector<point_info_t>* points) final;
+        void     getIndexFile        (const OGRGeometry* geo, std::string& file) final;
+        void     getIndexFile        (const std::vector<point_info_t>* points, std::string& file) final;
         bool     findRasters         (raster_finder_t* finder) final;
 
-        void     getGroupSamples     (const rasters_group_t* rgroup, List<RasterSample*>& slist, uint32_t flags) final
+        void     getSerialGroupSamples(const rasters_group_t* rgroup, List<RasterSample*>& slist, uint32_t flags) final
                                      { _getGroupSamples(SERIAL, rgroup, &slist, flags);}
 
         uint32_t getBatchGroupSamples(const rasters_group_t* rgroup, List<RasterSample*>* slist, uint32_t flags, uint32_t pointIndx) final

diff --git a/datasets/pgc/package/PgcDemStripsRaster.cpp b/datasets/pgc/package/PgcDemStripsRaster.cpp
@@ -91,34 +91,32 @@ bool PgcDemStripsRaster::getFeatureDate(const OGRFeature* feature, TimeLib::gmt_
 /*----------------------------------------------------------------------------
  * getIndexFile
  *----------------------------------------------------------------------------*/
-void PgcDemStripsRaster::getIndexFile(const OGRGeometry* geo, std::string& file, const std::vector<point_info_t>* points)
+void PgcDemStripsRaster::getIndexFile(const OGRGeometry* geo, std::string& file)
 {
-    const OGRPolygon* poly = NULL;
-
-    if(geo == NULL && points == NULL)
+    if(geo == NULL)
     {
-        mlog(ERROR, "Both geo and points are NULL");
+        mlog(ERROR, "geo param is NULL");
         ssErrors |= SS_INDEX_FILE_ERROR;
         return;
     }
 
     /* Determine if we have a point */
-    if(geo && GdalRaster::ispoint(geo))
+    if(GdalRaster::ispoint(geo))
     {
         /* Only one index file for a point from one of the geocells */
         const OGRPoint* poi = geo->toPoint();
         _getIndexFile(poi->getX(), poi->getY(), file);
         return;
     }
 
-    /* Vector holding all geojson files from all geocells */
+    /* Vector holding all geojson index files from all geocells */
     std::vector<std::string> files;
 
     /* Determine if we have a polygon */
-    if(geo && GdalRaster::ispoly(geo))
+    if(GdalRaster::ispoly(geo))
     {
         OGREnvelope env;
-        poly = geo->toPolygon();
+        const OGRPolygon* poly = geo->toPolygon();
         poly->getEnvelope(&env);
 
         const double minx = floor(env.MinX);
@@ -141,25 +139,69 @@ void PgcDemStripsRaster::getIndexFile(const OGRGeometry* geo, std::string& file,
         mlog(INFO, "Found %ld geojson files in polygon", files.size());
     }
 
-    /* If we don't have a polygon but we have points get all files for the points */
-    if(poly == NULL && points != NULL)
+    /* If we have only one file, use it as the index file */
+    if(files.size() == 1)
     {
-        for(const auto& p : *points)
+        file = files[0];
+        return;
+    }
+
+    /* Combine all geojson files into a single file stored in vsimem */
+    if(!combinedGeoJSON.empty())
+    {
+        /* Remove previous combined geojson file */
+        VSIUnlink(combinedGeoJSON.c_str());
+    }
+
+    combinedGeoJSON = "/vsimem/" + GdalRaster::getUUID() + "_combined.geojson";
+    if(combineGeoJSONFiles(files))
+    {
+        /* Set the combined geojson file as the index file */
+        file = combinedGeoJSON;
+    }
+    else
+    {
+        mlog(ERROR, "Failed to combine geojson files");
+        ssErrors |= SS_INDEX_FILE_ERROR;
+    }
+}
+
+/*----------------------------------------------------------------------------
+ * getIndexFile
+ *----------------------------------------------------------------------------*/
+void PgcDemStripsRaster::getIndexFile(const std::vector<point_info_t>* points, std::string& file)
+{
+    if(points == NULL)
+    {
+        mlog(ERROR, "points param is NULL");
+        ssErrors |= SS_INDEX_FILE_ERROR;
+        return;
+    }
+
+    /* Get all geojson files for all points */
+    std::vector<std::string> files;
+    for(const auto& p : *points)
+    {
+        std::string newFile;
+        _getIndexFile(p.point.x, p.point.y, newFile);
+        if(!newFile.empty())
         {
-            std::string newFile;
-            _getIndexFile(p.point.x, p.point.y, newFile);
-            if(!newFile.empty())
-            {
-                files.push_back(newFile);
-            }
+            files.push_back(newFile);
         }
-        mlog(INFO, "Found %zu geojson files with %zu points", files.size(), points->size());
     }
+    mlog(INFO, "Found %zu geojson files with %zu points", files.size(), points->size());
 
     /* Remove any duplicate files */
     std::sort(files.begin(), files.end());
     files.erase(std::unique(files.begin(), files.end()), files.end());
 
+    /* If we have only one file, use it as the index file */
+    if(files.size() == 1)
+    {
+        file = files[0];
+        return;
+    }
+
     /* Combine all geojson files into a single file stored in vsimem */
     if(!combinedGeoJSON.empty())
     {

diff --git a/datasets/pgc/package/PgcDemStripsRaster.h b/datasets/pgc/package/PgcDemStripsRaster.h
@@ -53,7 +53,8 @@ class PgcDemStripsRaster: public GeoIndexedRaster
                  PgcDemStripsRaster (lua_State* L, RequestFields* rqst_parms, const char* key, const char* dem_name, const char* geo_suffix, GdalRaster::overrideCRS_t cb);
                 ~PgcDemStripsRaster (void) override;
         bool     getFeatureDate     (const OGRFeature* feature, TimeLib::gmt_time_t& gmtDate) final;
-        void     getIndexFile       (const OGRGeometry* geo, std::string& file, const std::vector<point_info_t>* points) final;
+        void     getIndexFile       (const OGRGeometry* geo, std::string& file) final;
+        void     getIndexFile       (const std::vector<point_info_t>* points, std::string& file) final;
         bool     findRasters        (raster_finder_t* finder) final;
 
     private:

diff --git a/datasets/usgs3dep/package/Usgs3dep1meterDemRaster.cpp b/datasets/usgs3dep/package/Usgs3dep1meterDemRaster.cpp
@@ -82,14 +82,22 @@ Usgs3dep1meterDemRaster::~Usgs3dep1meterDemRaster(void)
 /*----------------------------------------------------------------------------
  * getIndexFile
  *----------------------------------------------------------------------------*/
-void Usgs3dep1meterDemRaster::getIndexFile(const OGRGeometry* geo, std::string& file, const std::vector<point_info_t>* points)
+void Usgs3dep1meterDemRaster::getIndexFile(const OGRGeometry* geo, std::string& file)
 {
     static_cast<void>(geo);
-    static_cast<void>(points);
     file = indexFile;
     mlog(DEBUG, "Using %s", file.c_str());
 }
 
+/*----------------------------------------------------------------------------
+ * getIndexFile
+ *----------------------------------------------------------------------------*/
+void Usgs3dep1meterDemRaster::getIndexFile(const std::vector<point_info_t>* points, std::string& file)
+{
+    static_cast<void>(points);
+    file = indexFile;
+    mlog(DEBUG, "Using %s", file.c_str());
+}
 
 /*----------------------------------------------------------------------------
  * findRasters

diff --git a/datasets/usgs3dep/package/Usgs3dep1meterDemRaster.h b/datasets/usgs3dep/package/Usgs3dep1meterDemRaster.h
@@ -69,7 +69,8 @@ class Usgs3dep1meterDemRaster: public GeoIndexedRaster
                 Usgs3dep1meterDemRaster (lua_State* L, RequestFields* rqst_parms, const char* key);
                ~Usgs3dep1meterDemRaster (void) override;
 
-        void    getIndexFile     (const OGRGeometry* geo, std::string& file, const std::vector<point_info_t>* points) final;
+        void    getIndexFile     (const OGRGeometry* geo, std::string& file) final;
+        void    getIndexFile     (const std::vector<point_info_t>* points, std::string& file) final;
         bool    findRasters      (raster_finder_t* finder) final;
 
         static OGRErr overrideTargetCRS(OGRSpatialReference& target);

diff --git a/packages/geo/CMakeLists.txt b/packages/geo/CMakeLists.txt
@@ -31,6 +31,8 @@ if (GDAL_FOUND AND PROJ_FOUND AND TIFF_FOUND)
             ${CMAKE_CURRENT_LIST_DIR}/GdalRaster.cpp
             ${CMAKE_CURRENT_LIST_DIR}/GeoRaster.cpp
             ${CMAKE_CURRENT_LIST_DIR}/GeoIndexedRaster.cpp
+            ${CMAKE_CURRENT_LIST_DIR}/GeoIndexedRasterSerial.cpp
+            ${CMAKE_CURRENT_LIST_DIR}/GeoIndexedRasterBatch.cpp
             ${CMAKE_CURRENT_LIST_DIR}/GeoJsonRaster.cpp
             ${CMAKE_CURRENT_LIST_DIR}/GeoUserRaster.cpp
             ${CMAKE_CURRENT_LIST_DIR}/RasterObject.cpp