diff --git a/celery/extractions.py b/celery/extractions.py index 11afafc..418a387 100644 --- a/celery/extractions.py +++ b/celery/extractions.py @@ -8,6 +8,7 @@ import sys import time import requests +from uuid import uuid4 from osgeo import gdal, ogr, osr from zipfile import ZipFile from celery import Task @@ -40,6 +41,10 @@ PROCESS_TIMEOUT = env.get("PROCESS_TIMEOUT", 3600) +DEBUG_CUTLINE = env.get("DEBUG_CUTLINE", False) + +GDAL_CONFIG_GDAL_CACHEMAX = env.get("GDAL_CONFIG_GDAL_CACHEMAX", 512) + def get_current_datetime(): return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) @@ -335,6 +340,20 @@ def normalize_resampling(method): return method +def set_config_option(**opts): + for k, v in opts.items(): + gdal.SetConfigOption(k, str(v)) + logger.info("GDAL config Options %s is set to %s." % ( + k, str(gdal.GetConfigOption(k)))) + + +def unset_config_option(**opts): + for k, _ in opts.items(): + gdal.SetConfigOption(k, None) + logger.info("GDAL config Options %s is set to %s." % ( + k, str(gdal.GetConfigOption(k)))) + + # Aimed at being run under do_process_in_forked_process() def process_raster(process_func_args, gdal_callback, gdal_callback_data): (params, tmpdir) = process_func_args @@ -415,9 +434,8 @@ def process_raster(process_func_args, gdal_callback, gdal_callback_data): # Project footprint to target SRS footprint_geom.TransformTo(dst_srs) - debug_cutline = False - if debug_cutline: - cutline_filename = "/tmp/cutline.json" + if DEBUG_CUTLINE: + cutline_filename = "/tmp/cutline_%s.json" % str(uuid4())[:7] else: cutline_filename = "/vsimem/cutline.json" @@ -541,6 +559,12 @@ def process_raster(process_func_args, gdal_callback, gdal_callback_data): translate_options = "-of " + driver_name for option in driver_options: translate_options += " -co %s=%s" % (option, driver_options[option]) + + config_options = { + 'GDAL_CACHEMAX': GDAL_CONFIG_GDAL_CACHEMAX, + } + + set_config_option(**config_options) logger.info( "Invoking gdal_translate %s %s %s" % (tmp_vrt, out_filename, translate_options) @@ -552,6 +576,8 @@ def process_raster(process_func_args, gdal_callback, gdal_callback_data): callback=create_scaled_progress(0, pct_max, gdal_callback), callback_data=gdal_callback_data, ) + unset_config_option(**config_options) + success = ret_ds is not None gdal.Unlink(tmp_vrt) diff --git a/frontend/app.py b/frontend/app.py index 74b761d..a783a75 100644 --- a/frontend/app.py +++ b/frontend/app.py @@ -29,6 +29,7 @@ def _(x): # create the app: app = Flask(__name__) +application = app # GUNICORN logger = logging.getLogger(__name__) @@ -86,13 +87,17 @@ def upper(x): @app.route("/jobs", methods=["POST"]) def submit(): """Submit a new task from the JSon content in the POST payload""" + logger.debug("app.py::jobs_get: POST `/jobs`") try: req = json.loads(request.data.decode("UTF-8")) except Exception as e: + logger.exception(e) return make_json_load_error(e) if type(req) != dict: + logger.error("TypeError: POST data is not a `dict`: %s(%s)." % (type(req), str(req))) return make_error(_("Payload should be a JSON dictionary")) + logger.info("POST data is: %s(%s)." % (type(req), str(req))) required_parameters = [ ("user_id", [str]), @@ -100,8 +105,12 @@ def submit(): ] for (k, accepted_types) in required_parameters: if k not in req: + logger.error("AttributeError: '%s' is not in %s." % (k, str(req))) return missing_parameter_error(k, req) if type(req[k]) not in accepted_types: + logger.error( + "TypeError: type `%s` for value '%s' is not in accepted types: %s." % ( + type(req[k]), str(req[k]), str(accepted_types))) return invalid_parameter_type_error(k, req) optional_parameters = [ @@ -115,6 +124,9 @@ def submit(): ] for (k, accepted_types) in optional_parameters: if k in req and type(req[k]) not in accepted_types: + logger.error( + "TypeError: type `%s` for value '%s' is not in accepted types: %s." % ( + type(req[k]), str(req[k]), str(accepted_types))) return invalid_parameter_type_error(k, req) user_email_address = req["user_email_address"] @@ -626,6 +638,7 @@ def is_valid_task(result): @app.route("/jobs/", methods=["GET"]) def jobs_get(task_id): """Get the status of a task""" + logger.debug("app.py::jobs_get: GET `/jobs/%s`" % task_id) res = taskmanager.AsyncResult(task_id) @@ -668,6 +681,7 @@ def jobs_get(task_id): @app.route("/jobs/", methods=["PUT"]) def jobs_put(task_id): """Cancel a task (if {"status": "STOP_REQUESTED"} payload provided)""" + logger.debug("app.py::jobs_put: PUT `/jobs/%s`" % task_id) try: req = json.loads(request.data.decode("UTF-8")) @@ -723,6 +737,7 @@ def jobs_put(task_id): @app.route("/jobs//download", methods=["GET"]) def jobs_download_result(task_id): """Return the result of a successful task as a .zip attachement""" + logger.debug("app.py::jobs_download_result: GET `/jobs/%s/download`" % task_id) res = taskmanager.AsyncResult(task_id)