Limit number of open handles in the openjpeg tile source.

Fix an issue with float/int in Python 2.7.

sources/openjpeg/large_image_source_openjpeg/__init__.py

@@ -20,10 +20,10 @@
 import math
 import PIL.Image
 import six
-import threading
 import warnings
 
 from six import BytesIO
+from six.moves import queue
 from xml.etree import cElementTree
 
 from pkg_resources import DistributionNotFound, get_distribution
@@ -77,6 +77,7 @@ class OpenjpegFileTileSource(FileTileSource):
 
     _minTileSize = 256
     _maxTileSize = 512
+    _maxOpenHandles = 6
 
     def __init__(self, path, **kwargs):
         """
@@ -91,17 +92,19 @@ def __init__(self, path, **kwargs):
 
         self._largeImagePath = largeImagePath
         self._pixelInfo = {}
-        self._openjpegLock = threading.RLock()
         try:
             self._openjpeg = glymur.Jp2k(largeImagePath)
         except glymur.jp2box.InvalidJp2kError:
             raise TileSourceException('File cannot be opened via Glymur and OpenJPEG.')
-        self._openjpegHandles = [self._openjpeg]
+        self._openjpegHandles = queue.LifoQueue()
+        for _ in range(self._maxOpenHandles - 1):
+            self._openjpegHandles.put(None)
+        self._openjpegHandles.put(self._openjpeg)
         try:
             self.sizeY, self.sizeX = self._openjpeg.shape[:2]
         except IndexError:
             raise TileSourceException('File cannot be opened via Glymur and OpenJPEG.')
-        self.levels = self._openjpeg.codestream.segment[2].num_res + 1
+        self.levels = int(self._openjpeg.codestream.segment[2].num_res) + 1
         self._minlevel = 0
         self.tileWidth = self.tileHeight = 2 ** int(math.ceil(max(
             math.log(float(self.sizeX)) / math.log(2) - self.levels + 1,
@@ -111,8 +114,8 @@ def __init__(self, path, **kwargs):
         if self.tileWidth < self._minTileSize or self.tileWidth > self._maxTileSize:
             self.tileWidth = self.tileHeight = min(
                 self._maxTileSize, max(self._minTileSize, self.tileWidth))
-            self.levels = math.ceil(math.log(float(max(
-                self.sizeX, self.sizeY)) / self.tileWidth) / math.log(2)) + 1
+            self.levels = int(math.ceil(math.log(float(max(
+                self.sizeX, self.sizeY)) / self.tileWidth) / math.log(2))) + 1
             self._minlevel = self.levels - self._openjpeg.codestream.segment[2].num_res - 1
         self._getAssociatedImages()
 
@@ -193,7 +196,7 @@ def getAssociatedImagesList(self):
 
         :return: the list of image keys.
         """
-        return list(self._associatedImages.keys())
+        return list(sorted(self._associatedImages.keys()))
 
     def _readbox(self, box):
         if box.length > 16 * 1024 * 1024:
@@ -211,7 +214,7 @@ def _readbox(self, box):
     def getTile(self, x, y, z, pilImageAllowed=False, **kwargs):
         if z < 0 or z >= self.levels:
             raise TileSourceException('z layer does not exist')
-        step = 2 ** (self.levels - 1 - z)
+        step = int(2 ** (self.levels - 1 - z))
         x0 = x * step * self.tileWidth
         x1 = min((x + 1) * step * self.tileWidth, self.sizeX)
         y0 = y * step * self.tileHeight
@@ -222,19 +225,23 @@ def getTile(self, x, y, z, pilImageAllowed=False, **kwargs):
             raise TileSourceException('y is outside layer')
         scale = None
         if z < self._minlevel:
-            scale = 2 ** (self._minlevel - z)
-            step = 2 ** (self.levels - 1 - self._minlevel)
+            scale = int(2 ** (self._minlevel - z))
+            step = int(2 ** (self.levels - 1 - self._minlevel))
         # possible open the file multiple times so multiple threads can access
         # it concurrently.
-        with self._openjpegLock:
-            if not len(self._openjpegHandles):
-                self._openjpegHandles.append(glymur.Jp2k(self._largeImagePath))
-            openjpegHandle = self._openjpegHandles.pop()
+        while True:
+            try:
+                # A timeout prevents uniterupptable waits on some platforms
+                openjpegHandle = self._openjpegHandles.get(timeout=1.0)
+                break
+            except queue.Empty:
+                continue
+        if openjpegHandle is None:
+            openjpegHandle = glymur.Jp2k(self._largeImagePath)
         try:
             tile = openjpegHandle[y0:y1:step, x0:x1:step]
         finally:
-            with self._openjpegLock:
-                self._openjpegHandles.append(openjpegHandle)
+            self._openjpegHandles.put(openjpegHandle)
         mode = 'L'
         if len(tile.shape) == 3:
             mode = ['L', 'LA', 'RGB', 'RGBA'][tile.shape[2] - 1]

test/data/JK-kidney_B-gal_H3_4C_1-500sec.jp2.sha512

@@ -0,0 +1 @@
+38912884b07a626d61a61dfede497abc31e407772bf300a553de737cb2289cb55aa94b059d4a304269900a5a267170912fa95d3b8260571bdffc14b311d5ec61

test/test_source_openjpeg.py

@@ -17,3 +17,40 @@ def testTilesFromOpenJPEG():
     assert tileMetadata['levels'] == 6
     assert tileMetadata['magnification'] == 40
     utilities.checkTilesZXY(source, tileMetadata)
+
+
+def testAssociatedImagesFromOpenJPEG():
+    imagePath = utilities.externaldata('data/JK-kidney_B-gal_H3_4C_1-500sec.jp2.sha512')
+    source = large_image_source_openjpeg.OpenjpegFileTileSource(imagePath)
+
+    imageList = source.getAssociatedImagesList()
+    assert imageList == ['label', 'macro']
+    image, mimeType = source.getAssociatedImage('macro')
+    assert image[:len(utilities.JPEGHeader)] == utilities.JPEGHeader
+    # Test missing associated image
+    assert source.getAssociatedImage('nosuchimage') is None
+
+
+def testBelowLevelTilesFromOpenJPEG():
+    from large_image.cache_util import cachesClear
+
+    imagePath = utilities.externaldata('data/JK-kidney_B-gal_H3_4C_1-500sec.jp2.sha512')
+    origMin = large_image_source_openjpeg.OpenjpegFileTileSource._minTileSize
+    origMax = large_image_source_openjpeg.OpenjpegFileTileSource._maxTileSize
+    large_image_source_openjpeg.OpenjpegFileTileSource._minTileSize = 64
+    large_image_source_openjpeg.OpenjpegFileTileSource._maxTileSize = 64
+    # Clear the cache to make sure we use our required max tile size.
+    cachesClear()
+    source = large_image_source_openjpeg.OpenjpegFileTileSource(imagePath)
+    tileMetadata = source.getMetadata()
+
+    assert tileMetadata['tileWidth'] == 64
+    assert tileMetadata['tileHeight'] == 64
+    assert tileMetadata['sizeX'] == 16384
+    assert tileMetadata['sizeY'] == 14848
+    assert tileMetadata['levels'] == 9
+    assert tileMetadata['magnification'] == 40
+    utilities.checkTilesZXY(source, tileMetadata)
+    large_image_source_openjpeg.OpenjpegFileTileSource._minTileSize = origMin
+    large_image_source_openjpeg.OpenjpegFileTileSource._maxTileSize = origMax
+    cachesClear()