Merge pull request #1 from camptocamp/stale_error_handler

Stale tile error handler

test for latest version of pillow failed, see PR mapproxy#530

doc/sources.rst

@@ -242,6 +242,12 @@ Each status code takes the following options:
 
 You need to enable ``transparent`` for your source, if you use ``on_error`` responses with transparency.
 
+``authorize_stale``
+
+  Set this to ``True`` if MapProxy should serve in priority stale tiles present in cache. If the specified source error occurs, MapProxy will serve a stale tile which is still in cache instead of the error reponse, even if the tile in cache should be refreshed according to refresh_before date. Otherwise (``False``) MapProxy will serve the unicolor error response defined by the error handler if the source is faulty and the tile is not in cache, or is stale.
+
+You need to enable ``transparent`` for your source, if you use ``on_error`` responses with transparency.
+
 ::
 
   my_tile_source:
@@ -250,6 +256,10 @@ You need to enable ``transparent`` for your source, if you use ``on_error`` resp
       url: http://localhost:8080/service?
       layers: base
     on_error:
+      404:
+        response: 'transparent'
+        cache: False
+        authorize_stale: True
       500:
         response: '#ede9e3'
         cache: False

mapproxy/cache/tile.py

@@ -385,6 +385,11 @@ def _create_single_tile(self, tile):
                     else:
                         reraise_exception(e, sys.exc_info())
                 if not source: return []
+                if source.authorize_stale and self.is_stale(tile):
+                    # The configuration authorises blank tiles generated by the error_handler
+                    # to be replaced by stale tiles from cache.
+                    self.cache.load_tile(tile)
+                    return [tile]
                 if self.tile_mgr.image_opts != source.image_opts:
                     # call as_buffer to force conversion into cache format
                     source.as_buffer(self.tile_mgr.image_opts)

mapproxy/config/loader.py

@@ -611,11 +611,12 @@ def on_error_handler(self):
                 raise ConfigurationError("invalid error code %r in on_error", status_code)
             cacheable = response_conf.get('cache', False)
             color = response_conf.get('response', 'transparent')
+            authorize_stale = response_conf.get('authorize_stale', False)
             if color == 'transparent':
                 color = (255, 255, 255, 0)
             else:
                 color = parse_color(color)
-            error_handler.add_handler(status_code, color, cacheable)
+            error_handler.add_handler(status_code, color, cacheable, authorize_stale)
 
         return error_handler
 

mapproxy/config/spec.py

@@ -189,6 +189,7 @@ def validate_options(conf_dict):
     anything(): {
         required('response'): one_of([int], str),
         'cache': bool,
+        'authorize_stale': bool
     }
 }
 

mapproxy/image/__init__.py

@@ -89,7 +89,24 @@ def tiff_tags(self, img_size):
         return tags
 
 
-class ImageSource(object):
+class BaseImageSource(object):
+    """
+    Virtual parent class for ImageSource and BlankImageSource
+    """
+    def __init__(self):
+        raise Exception("Virtual class BaseImageSource, cannot be instanciated.")
+
+    def as_image(self):
+        raise Exception("Virtual class BaseImageSource, method as_image cannot be called.")
+
+    def as_buffer(self, image_opts=None, format=None, seekable=False):
+        raise Exception("Virtual class BaseImageSource, method as_buffer cannot be called.")
+
+    def close_buffers(self):
+        pass
+
+
+class ImageSource(BaseImageSource):
     """
     This class wraps either a PIL image, a file-like object, or a file name.
     You can access the result as an image (`as_image` ) or a file-like buffer
@@ -111,6 +128,7 @@ def __init__(self, source, size=None, image_opts=None, cacheable=True, georef=No
         self._size = size
         self.cacheable = cacheable
         self.georef = georef
+        self.authorize_stale = False
 
     @property
     def source(self):
@@ -238,7 +256,7 @@ def SubImageSource(source, size, offset, image_opts, cacheable=True):
     img.paste(subimg, offset)
     return ImageSource(img, size=size, image_opts=new_image_opts, cacheable=cacheable)
 
-class BlankImageSource(object):
+class BlankImageSource(BaseImageSource):
     """
     ImageSource for transparent or solid-color images.
     Implements optimized as_buffer() method.
@@ -249,6 +267,7 @@ def __init__(self, size, image_opts, cacheable=False):
         self._buf = None
         self._img = None
         self.cacheable = cacheable
+        self.authorize_stale = False
 
     def as_image(self):
         if not self._img:

mapproxy/source/error.py

@@ -20,19 +20,20 @@ class HTTPSourceErrorHandler(object):
 	def __init__(self):
 		self.response_error_codes = {}
 
-	def add_handler(self, http_code, color, cacheable=False):
-		self.response_error_codes[http_code] = (color, cacheable)
+	def add_handler(self, http_code, color, cacheable=False, authorize_stale=False):
+		self.response_error_codes[http_code] = (color, cacheable, authorize_stale)
 
 	def handle(self, status_code, query):
 		color = cacheable = None
 		if status_code in self.response_error_codes:
-			color, cacheable = self.response_error_codes[status_code]
+			color, cacheable, authorize_stale = self.response_error_codes[status_code]
 		elif 'other' in self.response_error_codes:
-			color, cacheable = self.response_error_codes['other']
+			color, cacheable, authorize_stale = self.response_error_codes['other']
 		else:
 			return None
 
 		transparent = len(color) == 4
 		image_opts = ImageOptions(bgcolor=color, transparent=transparent)
 		img_source = BlankImageSource(query.size, image_opts, cacheable=cacheable)
-		return img_source
+		img_source.authorize_stale = authorize_stale
+		return img_source

mapproxy/test/system/fixture/tileservice_refresh.yaml

@@ -17,6 +17,10 @@ layers:
     title: Direct Layer
     sources: [wms_cache_isotime]
 
+  - name: wms_cache_png
+    title: Direct Layer
+    sources: [wms_cache_png]
+
 caches:
   wms_cache:
     format: image/jpeg
@@ -30,9 +34,26 @@ caches:
     refresh_before:
       time: "2009-02-15T23:31:30"
 
+  wms_cache_png:
+    format: image/png
+    sources: [wms_source]
+    refresh_before:
+      seconds: 1
+
 sources:
   wms_source:
     type: wms
     req:
       url: http://localhost:42423/service
       layers: bar
+    on_error:
+      404:
+        response: 'transparent'
+        cache: False
+      405:
+        response: '#ff0000'
+        cache: False
+      406:
+        response: 'transparent'
+        cache: False
+        authorize_stale: True

mapproxy/test/system/test_refresh.py

@@ -108,3 +108,100 @@ def test_refresh_tile_mtime(self, app, cache_dir):
                 t3 = file_path.mtime()
             assert t2 == t1
             assert t3 > t2
+
+    def test_refresh_tile_source_error_no_stale(self, app, cache_dir):
+        source_request = {
+            "path": r"/service?LAYERs=bar&SERVICE=WMS&FORMAT=image%2Fpng"
+            "&REQUEST=GetMap&HEIGHT=256&SRS=EPSG%3A900913&styles="
+            "&VERSION=1.1.1&BBOX=-20037508.3428,-20037508.3428,0.0,0.0"
+            "&WIDTH=256"
+        }
+        with tmp_image((256, 256), format="png") as img:
+            expected_req = (
+                source_request,
+                {"body": img.read(), "headers": {"content-type": "image/png"}},
+            )
+            with mock_httpd(
+                ("localhost", 42423), [expected_req], bbox_aware_query_comparator=True
+            ):
+                resp = app.get("/tiles/wms_cache_png/1/0/0.png")
+                assert resp.content_type == "image/png"
+                img.seek(0)
+                assert resp.body == img.read()
+            resp = app.get("/tiles/wms_cache_png/1/0/0.png")
+            assert resp.content_type == "image/png"
+            img.seek(0)
+            assert resp.body == img.read()
+            # tile is expired after 1 sec, so it will be requested again from mock server
+            time.sleep(1.2)
+            expected_req = (
+                source_request,
+                {"body": "", "status": 404},
+            )
+            with mock_httpd(
+                    ("localhost", 42423), [expected_req], bbox_aware_query_comparator=True
+            ):
+                resp = app.get("/tiles/wms_cache_png/1/0/0.png")
+                assert resp.content_type == "image/png"
+                # error handler for 404 does not authorise stale tiles, so transparent tile will be rendered
+                resp_img = Image.open(BytesIO(resp.body))
+                # check response transparency
+                assert resp_img.getbands() == ('R', 'G', 'B', 'A')
+                assert resp_img.getextrema()[3] == (0, 0)
+
+            expected_req = (
+                source_request,
+                {"body": "", "status": 405},
+            )
+            with mock_httpd(
+                    ("localhost", 42423), [expected_req], bbox_aware_query_comparator=True
+            ):
+                resp = app.get("/tiles/wms_cache_png/1/0/0.png")
+                assert resp.content_type == "image/png"
+                # error handler for 405 does not authorise stale tiles, so red tile will be rendered
+                resp_img = Image.open(BytesIO(resp.body))
+                # check response red color
+                assert resp_img.getbands() == ('R', 'G', 'B')
+                assert resp_img.getextrema() == ((255, 255), (0, 0), (0, 0))
+
+    def test_refresh_tile_source_error_stale(self, app, cache_dir):
+        with tmp_image((256, 256), format="jpeg") as img:
+            expected_req = (
+                {
+                    "path": r"/service?LAYERs=bar&SERVICE=WMS&FORMAT=image%2Fjpeg"
+                    "&REQUEST=GetMap&HEIGHT=256&SRS=EPSG%3A900913&styles="
+                    "&VERSION=1.1.1&BBOX=-20037508.3428,-20037508.3428,0.0,0.0"
+                    "&WIDTH=256"
+                },
+                {"body": img.read(), "headers": {"content-type": "image/jpeg"}},
+            )
+            with mock_httpd(
+                ("localhost", 42423), [expected_req], bbox_aware_query_comparator=True
+            ):
+                resp = app.get("/tiles/wms_cache/1/0/0.jpeg")
+                assert resp.content_type == "image/jpeg"
+                img.seek(0)
+                assert resp.body == img.read()
+            resp = app.get("/tiles/wms_cache/1/0/0.jpeg")
+            assert resp.content_type == "image/jpeg"
+            img.seek(0)
+            assert resp.body == img.read()
+            # tile is expired after 1 sec, so it will be fetched again from mock server
+            time.sleep(1.2)
+            expected_req = (
+                {
+                    "path": r"/service?LAYERs=bar&SERVICE=WMS&FORMAT=image%2Fjpeg"
+                    "&REQUEST=GetMap&HEIGHT=256&SRS=EPSG%3A900913&styles="
+                    "&VERSION=1.1.1&BBOX=-20037508.3428,-20037508.3428,0.0,0.0"
+                    "&WIDTH=256"
+                },
+                {"body": "", "status": 406},
+            )
+            with mock_httpd(
+                    ("localhost", 42423), [expected_req], bbox_aware_query_comparator=True
+            ):
+                resp = app.get("/tiles/wms_cache/1/0/0.jpeg")
+                assert resp.content_type == "image/jpeg"
+                # Check that initial non empty img is served as a stale tile
+                img.seek(0)
+                assert resp.body == img.read()