Merge pull request #345 from ICESat2-SlideRule/ulimit8192

enabled gdal multirange, needs ulimit -n 8192
SlideRuleEarth · Oct 16, 2023 · 839c29f · 839c29f
2 parents 5d5d429 + 9fd8f5a
commit 839c29f
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 6 deletions.
diff --git a/packages/geo/geo.cpp b/packages/geo/geo.cpp
@@ -183,20 +183,16 @@ static void configGDAL(void)
      * which is not supported by a majority of servers (including AWS S3 or Google GCS).
      * SERIAL means that each range will be requested sequentially.
      * YES means that each range will be requested in parallel, using HTTP/2 multiplexing or several HTTP connections.
-     *
-     * NOTE: if set to YES, multirange reads cause errors when multiple reader threads read their data sets.
-     *       Perfromed tests with 200 reader threads each reading 16MB and on average 14 to 16 readers failed.
-     *       setting it to SERIAL fixed the problem.
      */
-    CPLSetConfigOption("GDAL_HTTP_MULTIRANGE", "SERIAL");
+    CPLSetConfigOption("GDAL_HTTP_MULTIRANGE", "YES");
 
     /*
      * Tells GDAL to merge consecutive range GET requests.
      * GDAL_HTTP_MERGE_CONSECUTIVE_RANGES=[YES/NO]: Defaults to NO.
      * Only applies when GDAL_HTTP_MULTIRANGE is YES.
      * Defines if ranges of a single ReadMultiRange() request that are consecutive should be merged into a single request.
      */
-    CPLSetConfigOption("GDAL_HTTP_MERGE_CONSECUTIVE_RANGES", "NO");
+    CPLSetConfigOption("GDAL_HTTP_MERGE_CONSECUTIVE_RANGES", "YES");
 
     /*
      * When set to YES, this attempts to download multiple range requests in parallel, reusing the same TCP connection.

diff --git a/targets/slideruleearth-aws/docker-compose.yml b/targets/slideruleearth-aws/docker-compose.yml
@@ -8,6 +8,9 @@ services:
     entrypoint: /usr/local/bin/sliderule /usr/local/etc/sliderule/server.lua /usr/local/etc/sliderule/config.json
     stdin_open: true # docker run -i
     tty: true # docker run -t
+    ulimits:
+      nofile:
+        soft: "8192"
     volumes:
       - /etc/ssl/certs:/etc/ssl/certs
       - /data:/data

diff --git a/targets/slideruleearth-aws/terraform/cluster/docker-compose-sliderule.yml b/targets/slideruleearth-aws/terraform/cluster/docker-compose-sliderule.yml
@@ -8,6 +8,9 @@ services:
     entrypoint: /usr/local/bin/sliderule /usr/local/etc/sliderule/server.lua /usr/local/etc/sliderule/config.json
     stdin_open: true # docker run -i
     tty: true # docker run -t
+    ulimits:
+      nofile:
+        soft: "8192"
     volumes:
       - /etc/ssl/certs:/etc/ssl/certs
       - /data:/data