Skip to content

Commit

Permalink
Add an openjpeg source using the glymur library.
Browse files Browse the repository at this point in the history
  • Loading branch information
manthey committed Oct 4, 2019
1 parent 5edf700 commit 0e359a2
Show file tree
Hide file tree
Showing 11 changed files with 372 additions and 39 deletions.
7 changes: 5 additions & 2 deletions .circleci/release_pypi.sh
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,13 @@ twine upload --verbose dist/*
cd "$ROOTPATH/sources/mapnik"
python setup.py sdist
twine upload --verbose dist/*
cd "$ROOTPATH/sources/openslide"
cd "$ROOTPATH/sources/ometiff"
python setup.py sdist
twine upload --verbose dist/*
cd "$ROOTPATH/sources/ometiff"
cd "$ROOTPATH/sources/openjpeg"
python setup.py sdist
twine upload --verbose dist/*
cd "$ROOTPATH/sources/openslide"
python setup.py sdist
twine upload --verbose dist/*
cd "$ROOTPATH/sources/pil"
Expand Down
5 changes: 3 additions & 2 deletions large_image/tilesource/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@
from pkg_resources import iter_entry_points

from .base import TileSource, FileTileSource, TileOutputMimeTypes, \
TILE_FORMAT_IMAGE, TILE_FORMAT_PIL, TILE_FORMAT_NUMPY, nearPowerOfTwo
TILE_FORMAT_IMAGE, TILE_FORMAT_PIL, TILE_FORMAT_NUMPY, nearPowerOfTwo, \
etreeToDict
from ..exceptions import TileGeneralException, TileSourceException, TileSourceAssetstoreException
from .. import config
from ..constants import SourcePriority
Expand Down Expand Up @@ -82,5 +83,5 @@ def getTileSource(*args, **kwargs):
'TileSource', 'FileTileSource',
'exceptions', 'TileGeneralException', 'TileSourceException', 'TileSourceAssetstoreException',
'TileOutputMimeTypes', 'TILE_FORMAT_IMAGE', 'TILE_FORMAT_PIL', 'TILE_FORMAT_NUMPY',
'AvailableTileSources', 'getTileSource', 'nearPowerOfTwo',
'AvailableTileSources', 'getTileSource', 'nearPowerOfTwo', 'etreeToDict',
]
35 changes: 33 additions & 2 deletions large_image/tilesource/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import PIL.ImageColor
import PIL.ImageDraw
import six
from collections import defaultdict
from six import BytesIO

from ..cache_util import getTileCache, strhash, methodcache
Expand Down Expand Up @@ -95,6 +96,37 @@ def _letterboxImage(image, width, height, fill):
return result


def etreeToDict(t):
"""
Convert an xml etree to a nested dictionary without schema names in the
keys.
@param t: an etree.
@returns: a python dictionary with the results.
"""
# Remove schema
tag = t.tag.split('}', 1)[1] if t.tag.startswith('{') else t.tag
d = {tag: {}}
children = list(t)
if children:
entries = defaultdict(list)
for entry in map(etreeToDict, children):
for k, v in six.iteritems(entry):
entries[k].append(v)
d = {tag: {k: v[0] if len(v) == 1 else v
for k, v in six.iteritems(entries)}}

if t.attrib:
d[tag].update({(k.split('}', 1)[1] if k.startswith('{') else k): v
for k, v in six.iteritems(t.attrib)})
text = (t.text or '').strip()
if text and len(d[tag]):
d[tag]['text'] = text
elif text:
d[tag] = text
return d


def nearPowerOfTwo(val1, val2, tolerance=0.02):
"""
Check if two values are different by nearly a power of two.
Expand Down Expand Up @@ -979,8 +1011,7 @@ def _pilFormatMatches(self, image, match=True, **kwargs):
# compatibility could be an issue.
return False

def _outputTile(self, tile, tileEncoding, x, y, z, pilImageAllowed=False,
**kwargs):
def _outputTile(self, tile, tileEncoding, x, y, z, pilImageAllowed=False, **kwargs):
"""
Convert a tile from a PIL image or image in memory to the desired
encoding.
Expand Down
1 change: 1 addition & 0 deletions requirements-dev.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ girder>=3.0.3
girder-jobs>=3.0.3
-e sources/dummy
-e sources/mapnik
-e sources/openjpeg
-e sources/openslide
-e sources/pil
-e sources/test
Expand Down
1 change: 1 addition & 0 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
'dummy': ['large-image-source-dummy'],
'mapnik': ['large-image-source-mapnik'],
'ometiff': ['large-image-source-ometiff'],
'openjpeg': ['large-image-source-openjpeg'],
'openslide': ['large-image-source-openslide'],
'pil': ['large-image-source-pil'],
'tiff': ['large-image-source-tiff'],
Expand Down
215 changes: 215 additions & 0 deletions sources/openjpeg/large_image_source_openjpeg/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,215 @@
# -*- coding: utf-8 -*-

##############################################################################
# Copyright Kitware Inc.
#
# Licensed under the Apache License, Version 2.0 ( the "License" );
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##############################################################################

import glymur
import math
import PIL.Image
import six
import threading
import warnings

from six import BytesIO
from xml.etree import cElementTree

from pkg_resources import DistributionNotFound, get_distribution

from large_image.cache_util import LruCacheMetaclass, methodcache
from large_image.constants import SourcePriority, TILE_FORMAT_PIL
from large_image.exceptions import TileSourceException
from large_image.tilesource import FileTileSource, etreeToDict


try:
__version__ = get_distribution(__name__).version
except DistributionNotFound:
# package is not installed
pass


warnings.filterwarnings('ignore', category=UserWarning, module='glymur')


@six.add_metaclass(LruCacheMetaclass)
class OpenjpegFileTileSource(FileTileSource):
"""
Provides tile access to SVS files and other files the openjpeg library can
read.
"""

cacheName = 'tilesource'
name = 'openjpegfile'
extensions = {
None: SourcePriority.MEDIUM,
'jp2': SourcePriority.PREFERRED,
'jpf': SourcePriority.PREFERRED,
'j2k': SourcePriority.PREFERRED,
'jpx': SourcePriority.PREFERRED,
}
mimeTypes = {
None: SourcePriority.FALLBACK,
'image/jp2': SourcePriority.PREFERRED,
'image/jpx': SourcePriority.PREFERRED,
}

_boxToTag = {
# In the few samples I've seen, both of these appear to be macro images
b'mig ': 'macro',
b'mag ': 'label',
# This contains a largish image
# b'psi ': 'other',
}
_xmlTag = b'mxl '

def __init__(self, path, **kwargs):
"""
Initialize the tile class. See the base class for other available
parameters.
:param path: a filesystem path for the tile source.
"""
super(OpenjpegFileTileSource, self).__init__(path, **kwargs)

largeImagePath = self._getLargeImagePath()

self._largeImagePath = largeImagePath
self._pixelInfo = {}
self._openjpegLock = threading.RLock()
try:
self._openjpeg = glymur.Jp2k(largeImagePath)
except glymur.jp2box.InvalidJp2kError:
raise TileSourceException('File cannot be opened via Glymur and OpenJPEG.')
try:
self.sizeY, self.sizeX = self._openjpeg.shape[:2]
except IndexError:
raise TileSourceException('File cannot be opened via Glymur and OpenJPEG.')
self.levels = self._openjpeg.codestream.segment[2].num_res + 1
self.tileWidth = self.tileHeight = 2 ** int(math.ceil(max(
math.log(float(self.sizeX)) / math.log(2) - self.levels + 1,
math.log(float(self.sizeY)) / math.log(2) - self.levels + 1)))
# read associated images and metadata from boxes
self._associatedImages = {}
for box in self._openjpeg.box:
if box.box_id == self._xmlTag or box.box_id in self._boxToTag:
data = self._readbox(box)
if data is None:
continue
if box.box_id == self._xmlTag:
self._parseMetadataXml(data)
continue
try:
self._associatedImages[self._boxToTag[box.box_id]] = PIL.Image.open(
BytesIO(data))
except Exception:
pass
if box.box_id == 'jp2c':
for segment in box.codestream.segment:
if segment.marker_id == 'CME' and hasattr(segment, 'ccme'):
self._parseMetadataXml(segment.ccme)

def getNativeMagnification(self):
"""
Get the magnification at a particular level.
:return: magnification, width of a pixel in mm, height of a pixel in mm.
"""
mm_x = self._pixelInfo.get('mm_x')
mm_y = self._pixelInfo.get('mm_y')
# Estimate the magnification if we don't have a direct value
mag = self._pixelInfo.get('magnification') or 0.01 / mm_x if mm_x else None
return {
'magnification': mag,
'mm_x': mm_x,
'mm_y': mm_y,
}

def _parseMetadataXml(self, meta):
if not isinstance(meta, six.string_types):
meta = meta.decode('utf8', 'ignore')
try:
xml = cElementTree.fromstring(meta)
except Exception:
return
self._description_xml = etreeToDict(xml)
xml = self._description_xml
try:
# Optrascan metadata
scanDetails = xml.get('ScanInfo', xml.get('EncodeInfo'))['ScanDetails']
mag = float(scanDetails['Magnification'])
# In microns; convert to mm
scale = float(scanDetails['PixelResolution']) * 1e-3
self._pixelInfo = {
'magnification': mag,
'mm_x': scale,
'mm_y': scale,
}
except Exception:
pass

def _getAssociatedImage(self, imageKey):
"""
Get an associated image in PIL format.
:param imageKey: the key of the associated image.
:return: the image in PIL format or None.
"""
return self._associatedImages.get(imageKey)

def getAssociatedImagesList(self):
"""
Return a list of associated images.
:return: the list of image keys.
"""
return list(self._associatedImages.keys())

def _readbox(self, box):
if box.length > 16 * 1024 * 1024:
return
try:
fp = open(self._largeImagePath, 'rb')
headerLength = 16
fp.seek(box.offset + headerLength)
data = fp.read(box.length - headerLength)
return data
except Exception:
pass

@methodcache()
def getTile(self, x, y, z, pilImageAllowed=False, **kwargs):
if z < 0 or z >= self.levels:
raise TileSourceException('z layer does not exist')
step = 2 ** (self.levels - 1 - z)
x0 = x * step * self.tileWidth
x1 = min((x + 1) * step * self.tileWidth, self.sizeX)
y0 = y * step * self.tileHeight
y1 = min((y + 1) * step * self.tileHeight, self.sizeY)
if x < 0 or x0 >= self.sizeX:
raise TileSourceException('x is outside layer')
if y < 0 or y0 >= self.sizeY:
raise TileSourceException('y is outside layer')
with self._openjpegLock:
tile = self._openjpeg[y0:y1:step, x0:x1:step]
mode = 'L'
if len(tile.shape) == 3:
mode = ['L', 'LA', 'RGB', 'RGBA'][tile.shape[2] - 1]
tile = PIL.Image.frombytes(mode, (tile.shape[1], tile.shape[0]), tile)
if tile.size != (self.tileWidth, self.tileHeight):
wrap = PIL.Image.new(mode, (self.tileWidth, self.tileHeight))
wrap.paste(tile, (0, 0))
tile = wrap
return self._outputTile(tile, TILE_FORMAT_PIL, x, y, z, pilImageAllowed, **kwargs)
30 changes: 30 additions & 0 deletions sources/openjpeg/large_image_source_openjpeg/girder_source.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
# -*- coding: utf-8 -*-

##############################################################################
# Copyright Kitware Inc.
#
# Licensed under the Apache License, Version 2.0 ( the "License" );
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##############################################################################

from girder_large_image.girder_tilesource import GirderTileSource
from . import OpenjpegFileTileSource


class OpenjpegGirderTileSource(OpenjpegFileTileSource, GirderTileSource):
"""
Provides tile access to Girder items with an SVS file or other files that
the openslide library can read.
"""

cacheName = 'tilesource'
name = 'openjpeg'
Loading

0 comments on commit 0e359a2

Please sign in to comment.