Refactor image conversion task.

.circleci/make_wheels.sh

@@ -16,7 +16,9 @@ cd "$ROOTPATH/girder"
 pip wheel . --no-deps -w ~/wheels && rm -rf build
 cd "$ROOTPATH/girder_annotation"
 pip wheel . --no-deps -w ~/wheels && rm -rf build
-cd "$ROOTPATH/tasks"
+cd "$ROOTPATH/utilities/converter"
+pip wheel . --no-deps -w ~/wheels && rm -rf build
+cd "$ROOTPATH/utilities/tasks"
 pip wheel . --no-deps -w ~/wheels && rm -rf build
 cd "$ROOTPATH/sources/bioformats"
 pip wheel . --no-deps -w ~/wheels && rm -rf build

.circleci/release_pypi.sh

@@ -12,7 +12,10 @@ twine upload --verbose dist/*
 cd "$ROOTPATH/girder_annotation"
 python setup.py sdist
 twine upload --verbose dist/*
-cd "$ROOTPATH/tasks"
+cd "$ROOTPATH/utilities/converter"
+python setup.py sdist
+twine upload --verbose dist/*
+cd "$ROOTPATH/utilities/tasks"
 python setup.py sdist
 twine upload --verbose dist/*
 cd "$ROOTPATH/sources/bioformats"

.dockerignore

@@ -5,5 +5,5 @@ girder
 girder_annotation
 large_image
 sources
-tasks
+utilities
 examples

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## Unreleased
 
+### Changes
+- The image conversion task has been split into two packages, large_image_converter and large_image_tasks.  The tasks module is used with Girder and Girder Worker for converting images and depends on the converter package.  The converter package can be used as a stand-alone command line tool (#518)
+
 ### Features
 - Added a `canRead` method to the core module (#512)
 
@@ -11,10 +14,9 @@
 - The openjpeg tile source can decode with parallelism (#511)
 - Geospatial tile sources are preferred for geospatial files (#512)
 - Support decoding JP2k compressed tiles in the tiff tile source (#514)
->>>>>>> For the tiff tile source, allow decoding jp2k tiles.
 
 ### Bug Fixes
-- Harden updates of the item view after making a large image (#508)
+- Harden updates of the item view after making a large image (#508, #515)
 - Tiles in an unexpected color mode weren't consistently adjusted (#510)
 
 ## Version 1.3.2

README.rst

@@ -20,12 +20,14 @@ Large Image consists of several Python modules designed to work together.  These
 - ``large-image``: The core module.
   You can specify extras_require of the name of any tile source included with this repository, ``sources`` for all of the tile sources in the repository, ``memcached`` for using memcached for tile caching, or ``all`` for all of the tile sources and memcached.
 
+- ``large-image-converter``: A utility for using pyvips and other libraries to convert images into pyramidal tiff files that can be read efficiently by large_image.
+
 - ``girder-large-image``: Large Image as a Girder_ 3.x plugin.
   You can specify extras_require of ``tasks`` to install a Girder Worker task that can convert otherwise unreadable images to pyramidal tiff files.
 
 - ``girder-large-image-annotation``: Annotations for large images as a Girder_ 3.x plugin.
 
-- ``large-image-tasks``: A utility for using pyvips to convert images into pyramidal tiff files that can be read efficiently by large_image.  This can be used by itself or with Girder Worker.
+- ``large-image-tasks``: A utility for running the converter via Girder Worker.
 
 - Tile sources:
 

docs/make_docs.sh

@@ -21,7 +21,8 @@ sphinx-apidoc -f -o source/large_image_source_openslide ../sources/openslide/lar
 sphinx-apidoc -f -o source/large_image_source_pil ../sources/pil/large_image_source_pil
 sphinx-apidoc -f -o source/large_image_source_test ../sources/test/large_image_source_test
 sphinx-apidoc -f -o source/large_image_source_tiff ../sources/tiff/large_image_source_tiff
-sphinx-apidoc -f -o source/large_image_tasks ../tasks/large_image_tasks
+sphinx-apidoc -f -o source/large_image_converter ../utilities/converter/large_image_converter
+sphinx-apidoc -f -o source/large_image_tasks ../utilities/tasks/large_image_tasks
 sphinx-apidoc -f -o source/girder_large_image ../girder/girder_large_image
 sphinx-apidoc -f -o source/girder_large_image_annotation ../girder_annotation/girder_large_image_annotation
 

docs/source/index.rst

@@ -26,6 +26,7 @@ large_image also works as a Girder plugin with optional annotation support.
    large_image_source_pil/modules
    large_image_source_test/modules
    large_image_source_tiff/modules
+   large_image_converter/modules
    large_image_tasks/modules
    girder_large_image/modules
    girder_large_image_annotation/modules

girder/girder_large_image/__init__.py

@@ -77,6 +77,8 @@ def _postUpload(event):
         del item['largeImage']['expected']
         item['largeImage']['fileId'] = fileObj['_id']
         item['largeImage']['sourceName'] = 'tiff'
+        if fileObj['name'].endswith('.geo.tiff'):
+            item['largeImage']['sourceName'] = 'gdal'
         Item().save(item)
 
 

girder/girder_large_image/models/image_item.py

@@ -17,10 +17,8 @@
 #############################################################################
 
 import json
-import os
 import pymongo
 import six
-import time
 
 from girder import logger
 from girder.constants import SortDir
@@ -54,11 +52,10 @@ def initialize(self):
         ], {})])
 
     def createImageItem(self, item, fileObj, user=None, token=None,
-                        createJob=True, notify=False):
+                        createJob=True, notify=False, **kwargs):
         # Using setdefault ensures that 'largeImage' is in the item
         if 'fileId' in item.setdefault('largeImage', {}):
-            # TODO: automatically delete the existing large file
-            raise TileGeneralException('Item already has a largeImage set.')
+            raise TileGeneralException('Item already has largeImage set.')
         if fileObj['itemId'] != item['_id']:
             raise TileGeneralException('The provided file must be in the '
                                        'provided item.')
@@ -75,30 +72,26 @@ def createImageItem(self, item, fileObj, user=None, token=None,
         sourceName = girder_tilesource.getGirderTileSourceName(item, fileObj)
         if sourceName:
             item['largeImage']['sourceName'] = sourceName
-        if not sourceName:
+        if not sourceName or createJob == 'always':
             if not createJob:
                 raise TileGeneralException(
                     'A job must be used to generate a largeImage.')
             # No source was successful
             del item['largeImage']['fileId']
-            job = self._createLargeImageJob(item, fileObj, user, token)
+            job = self._createLargeImageJob(item, fileObj, user, token, **kwargs)
             item['largeImage']['expected'] = True
             item['largeImage']['notify'] = notify
             item['largeImage']['originalId'] = fileObj['_id']
             item['largeImage']['jobId'] = job['_id']
         self.save(item)
         return job
 
-    def _createLargeImageJob(self, item, fileObj, user, token):
+    def _createLargeImageJob(self, item, fileObj, user, token, **kwargs):
         import large_image_tasks.tasks
         from girder_worker_utils.transforms.girder_io import GirderUploadToItem
         from girder_worker_utils.transforms.contrib.girder_io import GirderFileIdAllowDirect
         from girder_worker_utils.transforms.common import TemporaryDirectory
 
-        outputName = os.path.splitext(fileObj['name'])[0] + '.tiff'
-        if outputName == fileObj['name']:
-            outputName = (os.path.splitext(fileObj['name'])[0] + '.' +
-                          time.strftime('%Y%m%d-%H%M%S') + '.tiff')
         try:
             localPath = File().getLocalFilePath(fileObj)
         except (FilePathException, AttributeError):
@@ -111,11 +104,12 @@ def _createLargeImageJob(self, item, fileObj, user, token):
                 'task': 'createImageItem',
             }},
             inputFile=GirderFileIdAllowDirect(str(fileObj['_id']), fileObj['name'], localPath),
-            outputName=outputName,
+            inputName=fileObj['name'],
             outputDir=TemporaryDirectory(),
             girder_result_hooks=[
                 GirderUploadToItem(str(item['_id']), False),
-            ]
+            ],
+            **kwargs,
         )
         return job.job
 

girder/girder_large_image/rest/tiles.py

@@ -143,9 +143,22 @@ def __init__(self, apiRoot):
         .param('fileId', 'The ID of the source file containing the image. '
                          'Required if there is more than one file in the item.',
                required=False)
+        .param('force', 'Always use a job to create the large image.',
+               dataType='boolean', default=False, required=False)
         .param('notify', 'If a job is required to create the large image, '
                'a nofication can be sent when it is complete.',
                dataType='boolean', default=True, required=False)
+        .param('tileSize', 'Tile size', dataType='int', default=256,
+               required=False)
+        .param('compression', 'Internal compression format', required=False,
+               enum=['none', 'jpeg', 'deflate', 'lzw', 'zstd', 'packbits', 'webp'])
+        .param('quality', 'JPEG compression quality where 0 is small and 100 '
+               'is highest quality', dataType='int', default=90,
+               required=False)
+        .param('level', 'Compression level for deflate (zip) or zstd.',
+               dataType='int', required=False)
+        .param('predictor', 'Predictor for deflate (zip) or lzw.',
+               required=False, enum=['none', 'horizontal', 'float', 'yes'])
     )
     @access.user
     @loadmodel(model='item', map={'itemId': 'item'}, level=AccessType.WRITE)
@@ -164,6 +177,7 @@ def createTiles(self, item, params):
         try:
             return self.imageItemModel.createImageItem(
                 item, largeImageFile, user, token,
+                createJob='always' if self.boolParam('force', params, default=False) else True,
                 notify=self.boolParam('notify', params, default=True))
         except TileGeneralException as e:
             raise RestException(e.args[0])
@@ -671,7 +685,8 @@ def getTilesThumbnail(self, item, params):
                enum=['0', '1', '2'], dataType='int', default='0')
         .param('tiffCompression', 'Compression method when storing a TIFF '
                'image', required=False,
-               enum=['raw', 'tiff_lzw', 'jpeg', 'tiff_adobe_deflate'])
+               enum=['none', 'raw', 'lzw', 'tiff_lzw', 'jpeg', 'deflate',
+                     'tiff_adobe_deflate'])
         .param('style', 'JSON-encoded style string', required=False)
         .param('resample', 'If false, an existing level of the image is used '
                'for the histogram.  If true, the internal values are '

girder/test_girder/test_tiles_rest.py

@@ -138,7 +138,7 @@ def _createTestTiles(server, admin, params=None, info=None, error=None):
     return infoDict
 
 
-def _postTileViaHttp(server, admin, itemId, fileId, jobAction=None):
+def _postTileViaHttp(server, admin, itemId, fileId, jobAction=None, data=None):
     """
     When we know we need to process a job, we have to use an actual http
     request rather than the normal simulated request to cherrypy.  This is
@@ -148,6 +148,9 @@ def _postTileViaHttp(server, admin, itemId, fileId, jobAction=None):
     :param itemId: the id of the item with the file to process.
     :param fileId: the id of the file that should be processed.
     :param jobAction: if 'delete', delete the job immediately.
+    :param data: if not None, pass this as the data to the POST request.  If
+        specified, fileId is ignored (pass as part of the data dictionary if
+        it is required).
     :returns: metadata from the tile if the conversion was successful,
               False if it converted but didn't result in useable tiles, and
               None if it failed.
@@ -158,14 +161,14 @@ def _postTileViaHttp(server, admin, itemId, fileId, jobAction=None):
     }
     req = requests.post('http://127.0.0.1:%d/api/v1/item/%s/tiles' % (
         server.boundPort, itemId), headers=headers,
-        data={'fileId': fileId})
+        data={'fileId': fileId} if data is None else data)
     assert req.status_code == 200
     # If we ask to create the item again right away, we should be told that
     # either there is already a job running or the item has already been
     # added
     req = requests.post('http://127.0.0.1:%d/api/v1/item/%s/tiles' % (
         server.boundPort, itemId), headers=headers,
-        data={'fileId': fileId})
+        data={'fileId': fileId} if data is None else data)
     assert req.status_code == 400
     assert ('Item already has' in req.json()['message'] or
             'Item is scheduled' in req.json()['message'])
@@ -1159,3 +1162,21 @@ def testTilesFromMultipleDotName(boundServer, admin, fsAssetstore, girderWorker)
     assert tileMetadata['mm_x'] is None
     assert tileMetadata['mm_y'] is None
     _testTilesZXY(boundServer, admin, itemId, tileMetadata)
+
+
+@pytest.mark.usefixtures('unbindLargeImage')  # noqa
+@pytest.mark.usefixtures('girderWorker')  # noqa
+@pytest.mark.plugin('large_image')
+def testTilesForcedConversion(boundServer, admin, fsAssetstore, girderWorker):  # noqa
+    file = utilities.uploadExternalFile(
+        'data/landcover_sample_1000.tif.sha512', admin, fsAssetstore)
+    itemId = str(file['itemId'])
+    fileId = str(file['_id'])
+    # We should already have tile information.  Ask to delete it so we can
+    # force convert it
+    boundServer.request(path='/item/%s/tiles' % itemId, method='DELETE', user=admin)
+    # Ask to do a forced conversion
+    tileMetadata = _postTileViaHttp(boundServer, admin, itemId, None, data={'force': True})
+    assert tileMetadata['levels'] == 3
+    item = Item().load(itemId, force=True)
+    assert item['largeImage']['fileId'] != fileId

requirements-dev.txt

@@ -15,11 +15,16 @@ girder-jobs>=3.0.3
 -e sources/ometiff
 # must be after source/gdal
 -e sources/mapnik
-# Get both the girder and worker dependencies so tasks can be used stand-alone
--e tasks[girder,worker]
+# Don't specify extras for the converter; they are already present above
+-e utilities/converter
+# Girder and worker dependencies are already installed above
+-e utilities/tasks
 -e girder/.
 -e girder_annotation/.
 
+# Extras from main setup.py
+pylibmc>=1.5.1
+
 # External dependencies
 pip>=9
 tox

requirements-worker.txt

@@ -0,0 +1,25 @@
+-e sources/bioformats
+-e sources/dummy
+-e sources/gdal
+-e sources/nd2
+-e sources/openjpeg
+-e sources/openslide
+-e sources/pil
+-e sources/test
+-e sources/tiff
+# must be after sources/tiff
+-e sources/ometiff
+# must be after source/gdal
+-e sources/mapnik
+# Don't specify extras for the converter; they are already present above
+-e utilities/converter
+# Worker dependencies are already installed above
+-e utilities/tasks
+
+# Extras from main setup.py
+pylibmc>=1.5.1
+
+# External dependencies
+pip>=9
+
+

setup.cfg

@@ -29,7 +29,7 @@ source =
     ../girder/girder_large_image
     ../girder_annotation/girder_large_image_annotation
     ../sources/
-    ../tasks/
+    ../utilities/
     ../examples/
     ../build/tox/*/lib/*/site-packages/large_image/
 
@@ -42,7 +42,7 @@ include =
     girder/girder_large_image/*
     girder_annotation/girder_large_image_annotation/*
     sources/*
-    tasks/*
+    utilities/*
     examples/*
     build/tox/*/lib/*/site-packages/*large_image*/*
 parallel = True

sources/tiff/large_image_source_tiff/tiff_reader.py

@@ -222,13 +222,15 @@ def _validate(self):  # noqa
             raise ValidationTiffException(
                 'Only RGB and greyscale TIFF files are supported')
 
-        if self._tiffInfo.get('bitspersample') not in (8, 16):
+        if self._tiffInfo.get('bitspersample') not in (8, 16, 32, 64):
             raise ValidationTiffException(
                 'Only 8 and 16 bits-per-sample TIFF files are supported')
 
         if self._tiffInfo.get('sampleformat') not in {
                 None,  # default is still SAMPLEFORMAT_UINT
-                libtiff_ctypes.SAMPLEFORMAT_UINT}:
+                libtiff_ctypes.SAMPLEFORMAT_UINT,
+                libtiff_ctypes.SAMPLEFORMAT_INT,
+                libtiff_ctypes.SAMPLEFORMAT_IEEEFP}:
             raise ValidationTiffException(
                 'Only unsigned int sampled TIFF files are supported')
 
@@ -615,10 +617,27 @@ def _getUncompressedTile(self, tileNum):
                 libtiff_ctypes.ORIENTATION_RIGHTBOT,
                 libtiff_ctypes.ORIENTATION_LEFTBOT}:
             tw, th = th, tw
-        image = numpy.ctypeslib.as_array(
-            ctypes.cast(imageBuffer, ctypes.POINTER(
-                ctypes.c_uint16 if self._tiffInfo.get('bitspersample') == 16 else ctypes.c_uint8)),
-            (th, tw, self._tiffInfo.get('samplesperpixel')))
+        format = (
+            self._tiffInfo.get('bitspersample'),
+            self._tiffInfo.get('sampleformat') if self._tiffInfo.get(
+                'sampleformat') is not None else libtiff_ctypes.SAMPLEFORMAT_UINT)
+        formattbl = {
+            (8, libtiff_ctypes.SAMPLEFORMAT_UINT): numpy.uint8,
+            (8, libtiff_ctypes.SAMPLEFORMAT_INT): numpy.int8,
+            (16, libtiff_ctypes.SAMPLEFORMAT_UINT): numpy.uint16,
+            (16, libtiff_ctypes.SAMPLEFORMAT_INT): numpy.int16,
+            (16, libtiff_ctypes.SAMPLEFORMAT_IEEEFP): numpy.float16,
+            (32, libtiff_ctypes.SAMPLEFORMAT_UINT): numpy.uint32,
+            (32, libtiff_ctypes.SAMPLEFORMAT_INT): numpy.int32,
+            (32, libtiff_ctypes.SAMPLEFORMAT_IEEEFP): numpy.float32,
+            (64, libtiff_ctypes.SAMPLEFORMAT_UINT): numpy.uint64,
+            (64, libtiff_ctypes.SAMPLEFORMAT_INT): numpy.int64,
+            (64, libtiff_ctypes.SAMPLEFORMAT_IEEEFP): numpy.float64,
+        }
+        image = numpy.ctypeslib.as_array(ctypes.cast(
+            imageBuffer, ctypes.POINTER(ctypes.c_uint8)), (tileSize, )).view(
+                formattbl[format]).reshape(
+                    (th, tw, self._tiffInfo.get('samplesperpixel')))
         if (self._tiffInfo.get('samplesperpixel') == 3 and
                 self._tiffInfo.get('photometric') == libtiff_ctypes.PHOTOMETRIC_YCBCR):
             if self._tiffInfo.get('bitspersample') == 16:
@@ -766,7 +785,9 @@ def getTile(self, x, y):
         if (not self._tiffInfo.get('istiled') or
                 self._tiffInfo.get('compression') not in (
                     libtiff_ctypes.COMPRESSION_JPEG, 33003, 33005, 34712) or
-                self._tiffInfo.get('bitspersample') != 8):
+                self._tiffInfo.get('bitspersample') != 8 or
+                self._tiffInfo.get('sampleformat') not in {
+                    None, libtiff_ctypes.SAMPLEFORMAT_UINT}):
             return self._getUncompressedTile(tileNum)
 
         imageBuffer = six.BytesIO()