Merge pull request #10 from neogeo-technologies/develop

Improvements (#9)

celery/extractions.py

@@ -8,6 +8,7 @@
 import sys
 import time
 import requests
+from uuid import uuid4
 from osgeo import gdal, ogr, osr
 from zipfile import ZipFile
 from celery import Task
@@ -40,6 +41,10 @@
 
 PROCESS_TIMEOUT = env.get("PROCESS_TIMEOUT", 3600)
 
+DEBUG_CUTLINE = env.get("DEBUG_CUTLINE", False)
+
+GDAL_CONFIG_GDAL_CACHEMAX = env.get("GDAL_CONFIG_GDAL_CACHEMAX", 512)
+
 
 def get_current_datetime():
     return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
@@ -335,6 +340,20 @@ def normalize_resampling(method):
     return method
 
 
+def set_config_option(**opts):
+    for k, v in opts.items():
+        gdal.SetConfigOption(k, str(v))
+        logger.info("GDAL config Options %s is set to %s." % (
+            k, str(gdal.GetConfigOption(k))))
+
+
+def unset_config_option(**opts):
+    for k, _ in opts.items():
+        gdal.SetConfigOption(k, None)
+        logger.info("GDAL config Options %s is set to %s." % (
+            k, str(gdal.GetConfigOption(k))))
+
+
 # Aimed at being run under do_process_in_forked_process()
 def process_raster(process_func_args, gdal_callback, gdal_callback_data):
     (params, tmpdir) = process_func_args
@@ -415,9 +434,8 @@ def process_raster(process_func_args, gdal_callback, gdal_callback_data):
         # Project footprint to target SRS
         footprint_geom.TransformTo(dst_srs)
 
-        debug_cutline = False
-        if debug_cutline:
-            cutline_filename = "/tmp/cutline.json"
+        if DEBUG_CUTLINE:
+            cutline_filename = "/tmp/cutline_%s.json" % str(uuid4())[:7]
         else:
             cutline_filename = "/vsimem/cutline.json"
 
@@ -541,6 +559,12 @@ def process_raster(process_func_args, gdal_callback, gdal_callback_data):
         translate_options = "-of " + driver_name
         for option in driver_options:
             translate_options += " -co %s=%s" % (option, driver_options[option])
+
+        config_options = {
+            'GDAL_CACHEMAX': GDAL_CONFIG_GDAL_CACHEMAX,
+        }
+
+        set_config_option(**config_options)
         logger.info(
             "Invoking gdal_translate %s %s %s"
             % (tmp_vrt, out_filename, translate_options)
@@ -552,6 +576,8 @@ def process_raster(process_func_args, gdal_callback, gdal_callback_data):
             callback=create_scaled_progress(0, pct_max, gdal_callback),
             callback_data=gdal_callback_data,
         )
+        unset_config_option(**config_options)
+
         success = ret_ds is not None
         gdal.Unlink(tmp_vrt)
 

frontend/app.py

@@ -29,6 +29,7 @@ def _(x):
 
 # create the app:
 app = Flask(__name__)
+application = app  # GUNICORN
 
 logger = logging.getLogger(__name__)
 
@@ -86,22 +87,30 @@ def upper(x):
 @app.route("/jobs", methods=["POST"])
 def submit():
     """Submit a new task from the JSon content in the POST payload"""
+    logger.debug("app.py::jobs_get: POST `/jobs`")
 
     try:
         req = json.loads(request.data.decode("UTF-8"))
     except Exception as e:
+        logger.exception(e)
         return make_json_load_error(e)
     if type(req) != dict:
+        logger.error("TypeError: POST data is not a `dict`: %s(%s)." % (type(req), str(req)))
         return make_error(_("Payload should be a JSON dictionary"))
+    logger.info("POST data is: %s(%s)." % (type(req), str(req)))
 
     required_parameters = [
         ("user_id", [str]),
         ("user_email_address", [str]),
     ]
     for (k, accepted_types) in required_parameters:
         if k not in req:
+            logger.error("AttributeError: '%s' is not in %s." % (k, str(req)))
             return missing_parameter_error(k, req)
         if type(req[k]) not in accepted_types:
+            logger.error(
+                "TypeError: type `%s` for value '%s' is not in accepted types: %s." % (
+                    type(req[k]), str(req[k]), str(accepted_types)))
             return invalid_parameter_type_error(k, req)
 
     optional_parameters = [
@@ -115,6 +124,9 @@ def submit():
     ]
     for (k, accepted_types) in optional_parameters:
         if k in req and type(req[k]) not in accepted_types:
+            logger.error(
+                "TypeError: type `%s` for value '%s' is not in accepted types: %s." % (
+                    type(req[k]), str(req[k]), str(accepted_types)))
             return invalid_parameter_type_error(k, req)
 
     user_email_address = req["user_email_address"]
@@ -626,6 +638,7 @@ def is_valid_task(result):
 @app.route("/jobs/<string:task_id>", methods=["GET"])
 def jobs_get(task_id):
     """Get the status of a task"""
+    logger.debug("app.py::jobs_get: GET `/jobs/%s`" % task_id)
 
     res = taskmanager.AsyncResult(task_id)
 
@@ -668,6 +681,7 @@ def jobs_get(task_id):
 @app.route("/jobs/<string:task_id>", methods=["PUT"])
 def jobs_put(task_id):
     """Cancel a task (if {"status": "STOP_REQUESTED"} payload provided)"""
+    logger.debug("app.py::jobs_put: PUT `/jobs/%s`" % task_id)
 
     try:
         req = json.loads(request.data.decode("UTF-8"))
@@ -723,6 +737,7 @@ def jobs_put(task_id):
 @app.route("/jobs/<string:task_id>/download", methods=["GET"])
 def jobs_download_result(task_id):
     """Return the result of a successful task as a .zip attachement"""
+    logger.debug("app.py::jobs_download_result: GET `/jobs/%s/download`" % task_id)
 
     res = taskmanager.AsyncResult(task_id)