Merge pull request #522 from girder/jp2k-tiles

Add support for converting to jp2k compression

CHANGELOG.md

@@ -7,6 +7,7 @@
 
 ### Features
 - Added a `canRead` method to the core module (#512)
+- Image conversion supports JPEG 2000 (jp2k) compression (#522)
 
 ### Improvements
 - Better release bioformats resources (#502)

girder/girder_large_image/rest/tiles.py

@@ -151,14 +151,19 @@ def __init__(self, apiRoot):
         .param('tileSize', 'Tile size', dataType='int', default=256,
                required=False)
         .param('compression', 'Internal compression format', required=False,
-               enum=['none', 'jpeg', 'deflate', 'lzw', 'zstd', 'packbits', 'webp'])
+               enum=['none', 'jpeg', 'deflate', 'lzw', 'zstd', 'packbits', 'webp', 'jp2k'])
         .param('quality', 'JPEG compression quality where 0 is small and 100 '
                'is highest quality', dataType='int', default=90,
                required=False)
         .param('level', 'Compression level for deflate (zip) or zstd.',
                dataType='int', required=False)
         .param('predictor', 'Predictor for deflate (zip) or lzw.',
                required=False, enum=['none', 'horizontal', 'float', 'yes'])
+        .param('psnr', 'JP2K compression target peak-signal-to-noise-ratio '
+               'where 0 is lossless and otherwise higher numbers are higher '
+               'quality', dataType='int', required=False)
+        .param('cr', 'JP2K target compression ratio where 1 is lossless',
+               dataType='int', required=False)
     )
     @access.user
     @loadmodel(model='item', map={'itemId': 'item'}, level=AccessType.WRITE)

test/test_converter.py

@@ -5,10 +5,12 @@
 import shutil
 import tifftools
 
+from large_image import constants
+import large_image_source_tiff
+
 import large_image_converter
 import large_image_converter.__main__ as main
 
-
 from . import utilities
 
 
@@ -129,6 +131,29 @@ def testConvertTiffFloatPixels(tmpdir):
             tifftools.constants.SampleFormat.uint.value)
 
 
+def testConvertJp2kCompression(tmpdir):
+    imagePath = utilities.externaldata('data/sample_Easy1.png.sha512')
+    outputPath = os.path.join(tmpdir, 'out.tiff')
+    large_image_converter.convert(imagePath, outputPath, compression='jp2k')
+    info = tifftools.read_tiff(outputPath)
+    assert (info['ifds'][0]['tags'][tifftools.Tag.Compression.value]['data'][0] ==
+            tifftools.constants.Compression.JP2000.value)
+    source = large_image_source_tiff.TiffFileTileSource(outputPath)
+    image, _ = source.getRegion(
+        output={'maxWidth': 200, 'maxHeight': 200}, format=constants.TILE_FORMAT_NUMPY)
+    assert (image[12][167] == [215, 135, 172]).all()
+
+    outputPath2 = os.path.join(tmpdir, 'out2.tiff')
+    large_image_converter.convert(imagePath, outputPath2, compression='jp2k', psnr=50)
+    assert os.path.getsize(outputPath2) < os.path.getsize(outputPath)
+
+    outputPath3 = os.path.join(tmpdir, 'out3.tiff')
+    large_image_converter.convert(imagePath, outputPath3, compression='jp2k', cr=100)
+    assert os.path.getsize(outputPath3) < os.path.getsize(outputPath)
+    assert os.path.getsize(outputPath3) != os.path.getsize(outputPath2)
+    # ##DWM::
+
+
 def testConverterMain(tmpdir):
     testDir = os.path.dirname(os.path.realpath(__file__))
     imagePath = os.path.join(testDir, 'test_files', 'yb10kx5k.png')

utilities/converter/large_image_converter/__init__.py

@@ -4,9 +4,11 @@
 import logging
 import os
 from pkg_resources import DistributionNotFound, get_distribution
+import struct
 from tempfile import TemporaryDirectory
 import time
 
+import numpy
 import tifftools
 
 try:
@@ -117,8 +119,8 @@ def _generate_tiff(inputPath, outputPath, tempPath, lidata, **kwargs):
         images.
     Optional parameters that can be specified in kwargs:
     :params tileSize: the horizontal and vertical tile size.
-    :param compression: one of 'jpeg', 'deflate' (zip), 'lzw', 'packbits', or
-        'zstd'.
+    :param compression: one of 'jpeg', 'deflate' (zip), 'lzw', 'packbits',
+        'zstd', or 'jp2k'.
     :params quality: a jpeg quality passed to vips.  0 is small, 100 is high
         quality.  90 or above is recommended.
     :param level: compression level for zstd, 1-22 (default is 10).
@@ -159,6 +161,111 @@ def _convert_via_vips(inputPathOrBuffer, outputPath, tempPath, forTiled=True,
             os.environ['TMPDIR'] = oldtmpdir
         else:
             del os.environ['TMPDIR']
+    if kwargs.get('compression') == 'jp2k':
+        _convert_to_jp2k(outputPath, **kwargs)
+
+
+def _convert_to_jp2k_tile(lock, fptr, dest, offset, length, shape, dtype, jp2kargs):
+    """
+    Read an uncompressed tile from a file and save it as a JP2000 file.
+
+    :param lock: a lock to ensure exclusive access to the file.
+    :param fptr: a pointer to the open file.
+    :param dest: the output path for the jp2k file.
+    :param offset: the location in the input file with the data.
+    :param length: the number of bytes to read.
+    :param shape: a tuple with the shape of the tile to read.  This is usually
+        (height, width, channels).
+    :param dtype: the numpy dtype of the data in the tile.
+    :param jp2kargs: arguments to pass to the compression, such as psnr or
+        cratios.
+    """
+    import glymur
+
+    with lock:
+        fptr.seek(offset)
+        data = fptr.read(length)
+    data = numpy.frombuffer(data, dtype=dtype)
+    data = numpy.reshape(data, shape)
+    glymur.Jp2k(dest, data=data, **jp2kargs)
+
+
+def _convert_to_jp2k(path, **kwargs):
+    """
+    Given a tiled tiff file without compression, convert it to jp2k compression
+    using the gylmur library.  This expects a tiff as written by vips without
+    any subifds.
+
+    :param path: the path of the tiff file.  The file is altered.
+    :param psnr: if set, the target psnr.  0 for lossless.
+    :param cr: is set, the target compression ratio.  1 for lossless.
+    """
+    import concurrent.futures
+    import psutil
+    import threading
+
+    info = tifftools.read_tiff(path)
+    jp2kargs = {}
+    if 'psnr' in kwargs:
+        jp2kargs['psnr'] = [int(kwargs['psnr'])]
+    elif 'cr' in kwargs:
+        jp2kargs['cratios'] = [int(kwargs['cr'])]
+    tilecount = sum(len(ifd['tags'][tifftools.Tag.TileOffsets.value]['data'])
+                    for ifd in info['ifds'])
+    processed = 0
+    lastlog = 0
+    tasks = []
+    lock = threading.Lock()
+    concurrency = psutil.cpu_count(logical=True)
+    pool = concurrent.futures.ThreadPoolExecutor(max_workers=concurrency)
+    with open(path, 'r+b') as fptr:
+        for ifd in info['ifds']:
+            ifd['tags'][tifftools.Tag.Compression.value]['data'][0] = (
+                tifftools.constants.Compression.JP2000)
+            shape = (
+                ifd['tags'][tifftools.Tag.TileHeight.value]['data'][0],
+                ifd['tags'][tifftools.Tag.TileLength.value]['data'][0],
+                len(ifd['tags'][tifftools.Tag.BitsPerSample.value]['data']))
+            dtype = numpy.uint16 if ifd['tags'][
+                tifftools.Tag.BitsPerSample.value]['data'][0] == 16 else numpy.uint8
+            for idx, offset in enumerate(ifd['tags'][tifftools.Tag.TileOffsets.value]['data']):
+                tmppath = path + '%d.jp2k' % processed
+                tasks.append((ifd, idx, processed, tmppath, pool.submit(
+                    _convert_to_jp2k_tile, lock, fptr, tmppath, offset,
+                    ifd['tags'][tifftools.Tag.TileByteCounts.value]['data'][idx],
+                    shape, dtype, jp2kargs)))
+                processed += 1
+        while len(tasks):
+            try:
+                tasks[0][-1].result(0.1)
+            except concurrent.futures.TimeoutError:
+                continue
+            ifd, idx, processed, tmppath, task = tasks.pop(0)
+            data = open(tmppath, 'rb').read()
+            os.unlink(tmppath)
+            # Remove first comment marker.  It adds needless bytes
+            compos = data.find(b'\xff\x64')
+            if compos >= 0 and compos + 4 < len(data):
+                comlen = struct.unpack('>H', data[compos + 2:compos + 4])[0]
+                if compos + 2 + comlen + 1 < len(data) and data[compos + 2 + comlen] == 0xff:
+                    data = data[:compos] + data[compos + 2 + comlen:]
+            with lock:
+                fptr.seek(0, os.SEEK_END)
+                ifd['tags'][tifftools.Tag.TileOffsets.value]['data'][idx] = fptr.tell()
+                ifd['tags'][tifftools.Tag.TileByteCounts.value]['data'][idx] = len(data)
+                fptr.write(data)
+            if time.time() - lastlog >= 10 and tilecount > 1:
+                logger.debug('Converted %d of %d tiles to jp2k', processed + 1, tilecount)
+                lastlog = time.time()
+        pool.shutdown(False)
+        fptr.seek(0, os.SEEK_END)
+        for ifd in info['ifds']:
+            ifd['size'] = fptr.tell()
+        info['size'] = fptr.tell()
+    tmppath = path + '.jp2k.tiff'
+    tifftools.write_tiff(info, tmppath, bigtiff=False, allowExisting=True)
+    os.unlink(path)
+    os.rename(tmppath, path)
 
 
 def _output_tiff(inputs, outputPath, lidata, extraImages=None):
@@ -394,6 +501,8 @@ def _vips_parameters(forTiled=True, **kwargs):
     }.items():
         if kwkey in kwargs and kwargs[kwkey] not in {None, ''}:
             convertParams[vkey] = kwargs[kwkey]
+    if convertParams['compression'] == 'jp2k':
+        convertParams['compression'] = 'none'
     if convertParams['predictor'] == 'yes':
         convertParams['predictor'] = 'horizontal'
     if convertParams['compression'] == 'jpeg':

utilities/converter/large_image_converter/__main__.py

@@ -24,12 +24,14 @@ def get_parser():
         '--compression', '-c',
         choices=[
             '', 'jpeg', 'deflate', 'zip', 'lzw', 'zstd', 'packbits', 'jbig',
-            'lzma', 'webp', 'none',
+            'lzma', 'webp', 'jp2k', 'none',
         ],
         help='Internal compression.  Default will use jpeg if the source '
         'appears to be lossy or lzw if lossless.  lzw is the most compatible '
         'lossless mode.  jpeg is the most compatible lossy mode.  jbig and '
-        'lzma may not be available.')
+        'lzma may not be available.  jp2k will first write the file with no '
+        'compression and then rewrite it with jp2k the specified psnr or '
+        'compression ratio.')
     parser.add_argument(
         '--quality', '-q', default=90, type=int,
         help='JPEG compression quality')
@@ -41,6 +43,11 @@ def get_parser():
         '--predictor', '-p', choices=['', 'none', 'horizontal', 'float', 'yes'],
         help='Predictor for some compressions.  Default is horizontal for '
         'non-geospatial data and yes for geospatial.')
+    parser.add_argument(
+        '--psnr', type=int,
+        help='JP2K peak signal to noise ratio.  0 for lossless')
+    parser.add_argument(
+        '--cr', type=int, help='JP2K compression ratio.  1 for lossless')
     parser.add_argument(
         '--tile', '-t', type=int, default=256, help='Tile size',
         dest='tileSize')