From 033b5e6134922caf0d510490bda50c91d0de36f1 Mon Sep 17 00:00:00 2001 From: David Manthey Date: Wed, 4 May 2022 11:11:11 -0400 Subject: [PATCH] General handling of skipped levels When a tile source is missing many levels of tiles, it can use too much memory to read the next available resolution and scale from that directly. This does that scaling in stages, if necessary. --- CHANGELOG.md | 7 +- large_image/tilesource/base.py | 68 +++++++++++++++- .../large_image_source_bioformats/__init__.py | 43 ++++------ .../large_image_source_dicom/__init__.py | 2 + .../large_image_source_openjpeg/__init__.py | 54 ++++++++----- .../large_image_source_openslide/__init__.py | 37 ++++++--- .../tiff/large_image_source_tiff/__init__.py | 69 ++-------------- .../large_image_source_tifffile/__init__.py | 81 ++++++++++++++----- .../zarr/large_image_source_zarr/__init__.py | 69 ++++++++++------ test/datastore.py | 2 + 10 files changed, 262 insertions(+), 170 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b4c74bfe..5fb47c329 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,12 @@ - Configurable item list grid view ([#1363](../../pull/1363)) - Allow labels in item list view ([#1366](../../pull/1366)) - Improve cache key guard ([#1368](../../pull/1368)) -- Improve handling dicom files in the working directory ([#1370](../../pull/137068)) +- Improve handling dicom files in the working directory ([#1370](../../pull/1370)) +- General handling of skipped levels ([#1373](../../pull/1373)) + +### Changes +- Update WsiDicomWebClient init call ([#1371](../../pull/1371)) +- Rename DICOMweb AssetstoreImportView ([#1372](../../pull/1372)) ### Bug Fixes - Default to "None" for the DICOM assetstore limit ([#1359](../../pull/1359)) diff --git a/large_image/tilesource/base.py b/large_image/tilesource/base.py index 89d985f16..24c64264d 100644 --- a/large_image/tilesource/base.py +++ b/large_image/tilesource/base.py @@ -57,6 +57,14 @@ class TileSource(IPyLeafletMixin): geospatial = False + # When getting tiles for otherwise empty levels (missing powers of two), we + # composite the tile from higher resolution levels. This can use excessive + # memory if there are too many missing levels. For instance, if there are + # six missing levels and the tile size is 1024 square RGBA, then 16 Gb are + # needed for the composited tile at a minimum. By setting + # _maxSkippedLevels, such large gaps are composited in stages. + _maxSkippedLevels = 3 + def __init__(self, encoding='JPEG', jpegQuality=95, jpegSubsampling=0, tiffCompression='raw', edge=False, style=None, noCache=None, *args, **kwargs): @@ -1924,6 +1932,54 @@ def _xyzToCorners(self, x, y, z): y1 = min((y + 1) * step * self.tileHeight, self.sizeY) return x0, y0, x1, y1, step + def _nonemptyLevelsList(self, frame=0): + """ + Return a list of one value per level where the value is None if the + level does not exist in the file and any other value if it does. + + :param frame: the frame number. + :returns: a list of levels length. + """ + return [True] * self.levels + + def _getTileFromEmptyLevel(self, x, y, z, **kwargs): + """ + Given the x, y, z tile location in an unpopulated level, get tiles from + higher resolution levels to make the lower-res tile. + + :param x: location of tile within original level. + :param y: location of tile within original level. + :param z: original level. + :returns: tile in PIL format. + """ + basez = z + scale = 1 + dirlist = self._nonemptyLevelsList(kwargs.get('frame')) + while dirlist[z] is None: + scale *= 2 + z += 1 + while z - basez > self._maxSkippedLevels: + z -= self._maxSkippedLevels + scale = int(scale / 2 ** self._maxSkippedLevels) + tile = PIL.Image.new('RGBA', ( + min(self.sizeX, self.tileWidth * scale), min(self.sizeY, self.tileHeight * scale))) + maxX = 2.0 ** (z + 1 - self.levels) * self.sizeX / self.tileWidth + maxY = 2.0 ** (z + 1 - self.levels) * self.sizeY / self.tileHeight + for newX in range(scale): + for newY in range(scale): + if ((newX or newY) and ((x * scale + newX) >= maxX or + (y * scale + newY) >= maxY)): + continue + subtile = self.getTile( + x * scale + newX, y * scale + newY, z, + pilImageAllowed=True, numpyAllowed=False, + sparseFallback=True, edge=False, frame=kwargs.get('frame')) + subtile = _imageToPIL(subtile) + tile.paste(subtile, (newX * self.tileWidth, + newY * self.tileHeight)) + return tile.resize((self.tileWidth, self.tileHeight), + getattr(PIL.Image, 'Resampling', PIL.Image).LANCZOS) + @methodcache() def getTile(self, x, y, z, pilImageAllowed=False, numpyAllowed=False, sparseFallback=False, frame=None): @@ -1993,10 +2049,16 @@ def getPreferredLevel(self, level): :param level: desired level :returns level: a level with actual data that is no lower resolution. """ - metadata = self.getMetadata() - if metadata['levels'] is None: + if self.levels is None: return level - return max(0, min(level, metadata['levels'] - 1)) + level = max(0, min(level, self.levels - 1)) + baselevel = level + levelList = self._nonemptyLevelsList() + while levelList[level] is None and level < self.levels - 1: + level += 1 + while level - baselevel >= self._maxSkippedLevels: + level -= self._maxSkippedLevels + return level def convertRegionScale( self, sourceRegion, sourceScale=None, targetScale=None, diff --git a/sources/bioformats/large_image_source_bioformats/__init__.py b/sources/bioformats/large_image_source_bioformats/__init__.py index cc8ed7237..2e61e9426 100644 --- a/sources/bioformats/large_image_source_bioformats/__init__.py +++ b/sources/bioformats/large_image_source_bioformats/__init__.py @@ -480,6 +480,19 @@ def _computeMagnification(self): self._magnification['magnification'] = float(metadata[key]) break + def _nonemptyLevelsList(self, frame=0): + """ + Return a list of one value per level where the value is None if the + level does not exist in the file and any other value if it does. + + :param frame: the frame number. + :returns: a list of levels length. + """ + nonempty = [True if v is not None else None + for v in self._metadata['frameSeries'][0]['series']][:self.levels] + nonempty += [None] * (self.levels - len(nonempty)) + return nonempty[::-1] + def getNativeMagnification(self): """ Get the magnification at a particular level. @@ -537,35 +550,6 @@ def getInternalMetadata(self, **kwargs): """ return self._metadata - def _getTileFromEmptyLevel(self, x, y, z, **kwargs): - """ - Composite tiles from missing levels from larger levels in pieces to - avoid using too much memory. - """ - fac = int(2 ** self._maxSkippedLevels) - z += self._maxSkippedLevels - scale = 2 ** (self.levels - 1 - z) - result = None - for tx in range(fac - 1, -1, -1): - if x * fac + tx >= int(math.ceil(self.sizeX / self.tileWidth / scale)): - continue - for ty in range(fac - 1, -1, -1): - if y * fac + ty >= int(math.ceil(self.sizeY / self.tileHeight / scale)): - continue - tile = self.getTile( - x * fac + tx, y * fac + ty, z, pilImageAllowed=False, - numpyAllowed=True, **kwargs) - if result is None: - result = np.zeros(( - ty * fac + tile.shape[0], - tx * fac + tile.shape[1], - tile.shape[2]), dtype=tile.dtype) - result[ - ty * fac:ty * fac + tile.shape[0], - tx * fac:tx * fac + tile.shape[1], - ::] = tile - return result[::scale, ::scale, ::] - @methodcache() def getTile(self, x, y, z, pilImageAllowed=False, numpyAllowed=False, **kwargs): self._xyzInRange(x, y, z) @@ -601,6 +585,7 @@ def getTile(self, x, y, z, pilImageAllowed=False, numpyAllowed=False, **kwargs): if scale >= 2 ** self._maxSkippedLevels: tile = self._getTileFromEmptyLevel(x, y, z, **kwargs) + tile = large_image.tilesource.base._imageToNumpy(tile)[0] format = TILE_FORMAT_NUMPY else: with self._tileLock: diff --git a/sources/dicom/large_image_source_dicom/__init__.py b/sources/dicom/large_image_source_dicom/__init__.py index 9297fc531..fc37dc985 100644 --- a/sources/dicom/large_image_source_dicom/__init__.py +++ b/sources/dicom/large_image_source_dicom/__init__.py @@ -157,6 +157,8 @@ def __init__(self, path, **kwargs): self.levels = int(max(1, math.ceil(math.log( max(self.sizeX / self.tileWidth, self.sizeY / self.tileHeight)) / math.log(2)) + 1)) self._populatedLevels = len(self._dicom.levels) + # We need to detect which levels are functionally present if we want to + # return a sensible _nonemptyLevelsList def _open_wsi_dicom(self, path): if isinstance(path, dict): diff --git a/sources/openjpeg/large_image_source_openjpeg/__init__.py b/sources/openjpeg/large_image_source_openjpeg/__init__.py index f8ca22f7d..896ea25ef 100644 --- a/sources/openjpeg/large_image_source_openjpeg/__init__.py +++ b/sources/openjpeg/large_image_source_openjpeg/__init__.py @@ -33,6 +33,7 @@ from large_image.constants import TILE_FORMAT_NUMPY, SourcePriority from large_image.exceptions import TileSourceError, TileSourceFileNotFoundError from large_image.tilesource import FileTileSource, etreeToDict +from large_image.tilesource.utilities import _imageToNumpy try: __version__ = _importlib_version(__name__) @@ -158,6 +159,17 @@ def _getAssociatedImages(self): if getattr(subbox, 'icc_profile', None): self._iccprofiles = [subbox.icc_profile] + def _nonemptyLevelsList(self, frame=0): + """ + Return a list of one value per level where the value is None if the + level does not exist in the file and any other value if it does. + + :param frame: the frame number. + :returns: a list of levels length. + """ + return [True if self.levels - 1 - idx < self._populatedLevels else None + for idx in range(self.levels)] + def getNativeMagnification(self): """ Get the magnification at a particular level. @@ -243,26 +255,30 @@ def getTile(self, x, y, z, pilImageAllowed=False, numpyAllowed=False, **kwargs): self._xyzInRange(x, y, z) x0, y0, x1, y1, step = self._xyzToCorners(x, y, z) scale = None - if z < self._minlevel: - scale = int(2 ** (self._minlevel - z)) - step = int(2 ** (self.levels - 1 - self._minlevel)) - # possibly open the file multiple times so multiple threads can access - # it concurrently. - while True: + if self._minlevel - z > self._maxSkippedLevels: + tile = self._getTileFromEmptyLevel(x, y, z, **kwargs) + tile = _imageToNumpy(tile)[0] + else: + if z < self._minlevel: + scale = int(2 ** (self._minlevel - z)) + step = int(2 ** (self.levels - 1 - self._minlevel)) + # possibly open the file multiple times so multiple threads can access + # it concurrently. + while True: + try: + # A timeout prevents uninterupptable waits on some platforms + openjpegHandle = self._openjpegHandles.get(timeout=1.0) + break + except queue.Empty: + continue + if openjpegHandle is None: + openjpegHandle = glymur.Jp2k(self._largeImagePath) try: - # A timeout prevents uninterupptable waits on some platforms - openjpegHandle = self._openjpegHandles.get(timeout=1.0) - break - except queue.Empty: - continue - if openjpegHandle is None: - openjpegHandle = glymur.Jp2k(self._largeImagePath) - try: - tile = openjpegHandle[y0:y1:step, x0:x1:step] - finally: - self._openjpegHandles.put(openjpegHandle) - if scale: - tile = tile[::scale, ::scale] + tile = openjpegHandle[y0:y1:step, x0:x1:step] + finally: + self._openjpegHandles.put(openjpegHandle) + if scale: + tile = tile[::scale, ::scale] return self._outputTile(tile, TILE_FORMAT_NUMPY, x, y, z, pilImageAllowed, numpyAllowed, **kwargs) diff --git a/sources/openslide/large_image_source_openslide/__init__.py b/sources/openslide/large_image_source_openslide/__init__.py index 25588e8b3..488e238bf 100644 --- a/sources/openslide/large_image_source_openslide/__init__.py +++ b/sources/openslide/large_image_source_openslide/__init__.py @@ -225,6 +225,16 @@ def _getAvailableLevels(self, path): nearPowerOfTwo(levels[0]['height'], entry['height'])] return levels + def _nonemptyLevelsList(self, frame=0): + """ + Return a list of one value per level where the value is None if the + level does not exist in the file and any other value if it does. + + :param frame: the frame number. + :returns: a list of levels length. + """ + return [True if l['scale'] == 1 else None for l in self._svslevels] + def getNativeMagnification(self): """ Get the magnification at a particular level. @@ -284,18 +294,21 @@ def getTile(self, x, y, z, pilImageAllowed=False, numpyAllowed=False, **kwargs): # We ask to read an area that will cover the tile at the z level. The # scale we computed in the __init__ process for this svs level tells # how much larger a region we need to read. - try: - tile = self._openslide.read_region( - (offsetx, offsety), svslevel['svslevel'], - (self.tileWidth * svslevel['scale'], - self.tileHeight * svslevel['scale'])) - except openslide.lowlevel.OpenSlideError as exc: - raise TileSourceError( - 'Failed to get OpenSlide region (%r).' % exc) - # Always scale to the svs level 0 tile size. - if svslevel['scale'] != 1: - tile = tile.resize((self.tileWidth, self.tileHeight), - getattr(PIL.Image, 'Resampling', PIL.Image).LANCZOS) + if svslevel['scale'] > 2 ** self._maxSkippedLevels: + tile = self._getTileFromEmptyLevel(x, y, z, **kwargs) + else: + try: + tile = self._openslide.read_region( + (offsetx, offsety), svslevel['svslevel'], + (self.tileWidth * svslevel['scale'], + self.tileHeight * svslevel['scale'])) + except openslide.lowlevel.OpenSlideError as exc: + raise TileSourceError( + 'Failed to get OpenSlide region (%r).' % exc) + # Always scale to the svs level 0 tile size. + if svslevel['scale'] != 1: + tile = tile.resize((self.tileWidth, self.tileHeight), + getattr(PIL.Image, 'Resampling', PIL.Image).LANCZOS) return self._outputTile(tile, TILE_FORMAT_PIL, x, y, z, pilImageAllowed, numpyAllowed, **kwargs) diff --git a/sources/tiff/large_image_source_tiff/__init__.py b/sources/tiff/large_image_source_tiff/__init__.py index 2b827a503..8251d919a 100644 --- a/sources/tiff/large_image_source_tiff/__init__.py +++ b/sources/tiff/large_image_source_tiff/__init__.py @@ -72,14 +72,6 @@ class TiffFileTileSource(FileTileSource, metaclass=LruCacheMetaclass): 'image/x-ptif': SourcePriority.PREFERRED, } - # When getting tiles for otherwise empty directories (missing powers of - # two), we composite the tile from higher resolution levels. This can use - # excessive memory if there are too many missing levels. For instance, if - # there are six missing levels and the tile size is 1024 square RGBA, then - # 16 Gb are needed for the composited tile at a minimum. By setting - # _maxSkippedLevels, such large gaps are composited in stages. - _maxSkippedLevels = 3 - _maxAssociatedImageSize = 8192 def __init__(self, path, **kwargs): # noqa @@ -643,7 +635,7 @@ def getTile(self, x, y, z, pilImageAllowed=False, numpyAllowed=False, if dir is None: try: if not kwargs.get('inSparseFallback'): - tile = self.getTileFromEmptyDirectory(x, y, z, **kwargs) + tile = self._getTileFromEmptyLevel(x, y, z, **kwargs) else: raise IOTiffError('Missing z level %d' % z) except Exception: @@ -713,64 +705,19 @@ def getTileIOTiffError(self, x, y, z, pilImageAllowed=False, numpyAllowed, applyStyle=False, **kwargs) raise TileSourceError('Internal I/O failure: %s' % exception.args[0]) - def getTileFromEmptyDirectory(self, x, y, z, **kwargs): + def _nonemptyLevelsList(self, frame=0): """ - Given the x, y, z tile location in an unpopulated level, get tiles from - higher resolution levels to make the lower-res tile. + Return a list of one value per level where the value is None if the + level does not exist in the file and any other value if it does. - :param x: location of tile within original level. - :param y: location of tile within original level. - :param z: original level. - :returns: tile in PIL format. + :param frame: the frame number. + :returns: a list of levels length. """ - basez = z - scale = 1 dirlist = self._tiffDirectories - frame = self._getFrame(**kwargs) + frame = int(frame or 0) if frame > 0 and hasattr(self, '_frames'): dirlist = self._frames[frame]['dirs'] - while dirlist[z] is None: - scale *= 2 - z += 1 - while z - basez > self._maxSkippedLevels: - z -= self._maxSkippedLevels - scale = int(scale / 2 ** self._maxSkippedLevels) - tile = PIL.Image.new('RGBA', ( - min(self.sizeX, self.tileWidth * scale), min(self.sizeY, self.tileHeight * scale))) - maxX = 2.0 ** (z + 1 - self.levels) * self.sizeX / self.tileWidth - maxY = 2.0 ** (z + 1 - self.levels) * self.sizeY / self.tileHeight - for newX in range(scale): - for newY in range(scale): - if ((newX or newY) and ((x * scale + newX) >= maxX or - (y * scale + newY) >= maxY)): - continue - subtile = self.getTile( - x * scale + newX, y * scale + newY, z, - pilImageAllowed=True, numpyAllowed=False, - sparseFallback=True, edge=False, frame=frame) - if not isinstance(subtile, PIL.Image.Image): - subtile = PIL.Image.open(io.BytesIO(subtile)) - tile.paste(subtile, (newX * self.tileWidth, - newY * self.tileHeight)) - return tile.resize((self.tileWidth, self.tileHeight), - getattr(PIL.Image, 'Resampling', PIL.Image).LANCZOS) - - def getPreferredLevel(self, level): - """ - Given a desired level (0 is minimum resolution, self.levels - 1 is max - resolution), return the level that contains actual data that is no - lower resolution. - - :param level: desired level - :returns level: a level with actual data that is no lower resolution. - """ - level = max(0, min(level, self.levels - 1)) - baselevel = level - while self._tiffDirectories[level] is None and level < self.levels - 1: - level += 1 - while level - baselevel >= self._maxSkippedLevels: - level -= self._maxSkippedLevels - return level + return dirlist def getAssociatedImagesList(self): """ diff --git a/sources/tifffile/large_image_source_tifffile/__init__.py b/sources/tifffile/large_image_source_tifffile/__init__.py index 3f8f580fd..2fbce1776 100644 --- a/sources/tifffile/large_image_source_tifffile/__init__.py +++ b/sources/tifffile/large_image_source_tifffile/__init__.py @@ -443,6 +443,45 @@ def _getAssociatedImage(self, imageKey): ], range(len(source.axes))) return large_image.tilesource.base._imageToPIL(image) + def _nonemptyLevelsList(self, frame=0): + """ + Return a list of one value per level where the value is None if the + level does not exist in the file and any other value if it does. + + :param frame: the frame number. + :returns: a list of levels length. + """ + if frame is None: + frame = 0 + if hasattr(self, '_nonempty_levels_list') and frame in self._nonempty_levels_list: + return self._nonempty_levels_list[frame] + if len(self._series) > 1: + sidx = frame // self._basis['P'][0] + else: + sidx = 0 + series = self._tf.series[self._series[sidx]] + nonempty = [None] * self.levels + nonempty[self.levels - 1] = True + xidx = series.axes.index('X') + yidx = series.axes.index('Y') + with self._zarrlock: + if sidx not in self._zarrcache: + if len(self._zarrcache) > 10: + self._zarrcache = {} + za = zarr.open(series.aszarr(), mode='r') + hasgbs = hasattr(za[0], 'get_basic_selection') + self._zarrcache[sidx] = (za, hasgbs) + za, hasgbs = self._zarrcache[sidx] + for ll in range(1, len(series.levels)): + scale = round(math.log(max(za[0].shape[xidx] / za[ll].shape[xidx], + za[0].shape[yidx] / za[ll].shape[yidx])) / math.log(2)) + if 0 < scale < self.levels: + nonempty[self.levels - 1 - int(scale)] = True + if not hasattr(self, '_nonempty_levels_list'): + self._nonempty_levels_list = {} + self._nonempty_levels_list[frame] = nonempty + return nonempty + @methodcache() def getTile(self, x, y, z, pilImageAllowed=False, numpyAllowed=False, **kwargs): frame = self._getFrame(**kwargs) @@ -479,25 +518,29 @@ def getTile(self, x, y, z, pilImageAllowed=False, numpyAllowed=False, **kwargs): break else: bza = za - sel = [] - baxis = '' - for aidx, axis in enumerate(series.axes): - if axis == 'X': - sel.append(slice(x0, x1, step)) - baxis += 'X' - elif axis == 'Y': - sel.append(slice(y0, y1, step)) - baxis += 'Y' - elif axis == 'S': - sel.append(slice(series.shape[aidx])) - baxis += 'S' - else: - sel.append((frame // self._basis[axis][0]) % self._basis[axis][2]) - tile = bza[tuple(sel)] - # rotate - if baxis not in {'YXS', 'YX'}: - tile = np.moveaxis( - tile, [baxis.index(a) for a in 'YXS' if a in baxis], range(len(baxis))) + if step > 2 ** self._maxSkippedLevels: + tile = self._getTileFromEmptyLevel(x, y, z, **kwargs) + tile = large_image.tilesource.base._imageToNumpy(tile)[0] + else: + sel = [] + baxis = '' + for aidx, axis in enumerate(series.axes): + if axis == 'X': + sel.append(slice(x0, x1, step)) + baxis += 'X' + elif axis == 'Y': + sel.append(slice(y0, y1, step)) + baxis += 'Y' + elif axis == 'S': + sel.append(slice(series.shape[aidx])) + baxis += 'S' + else: + sel.append((frame // self._basis[axis][0]) % self._basis[axis][2]) + tile = bza[tuple(sel)] + # rotate + if baxis not in {'YXS', 'YX'}: + tile = np.moveaxis( + tile, [baxis.index(a) for a in 'YXS' if a in baxis], range(len(baxis))) return self._outputTile(tile, TILE_FORMAT_NUMPY, x, y, z, pilImageAllowed, numpyAllowed, **kwargs) diff --git a/sources/zarr/large_image_source_zarr/__init__.py b/sources/zarr/large_image_source_zarr/__init__.py index 409e99ac6..39bfb4f8f 100644 --- a/sources/zarr/large_image_source_zarr/__init__.py +++ b/sources/zarr/large_image_source_zarr/__init__.py @@ -62,9 +62,9 @@ def __init__(self, path, **kwargs): try: self._zarr = zarr.open(self._largeImagePath, mode='r') except Exception: - if os.path.basename(self._largeImagePath) in {'.zgroup', '.zattrs'}: + if os.path.basename(self._largeImagePath) in {'.zgroup', '.zattrs', '.zarray'}: try: - self._zarr = zarr.open(os.path.dirname(self._largeImagePath)) + self._zarr = zarr.open(os.path.dirname(self._largeImagePath), mode='r') except Exception: pass if self._zarr is None: @@ -131,10 +131,10 @@ def _scanZarrArray(self, group, arr, results): associated images. These have to be culled for the actual groups used in the series. """ - attrs = group.attrs.asdict() + attrs = group.attrs.asdict() if group is not None else {} min_version = packaging.version.Version('0.4') is_ome = ( - isinstance(attrs['multiscales'], list) and + isinstance(attrs.get('multiscales', None), list) and 'omero' in attrs and isinstance(attrs['omero'], dict) and all(isinstance(m, dict) for m in attrs['multiscales']) and @@ -185,6 +185,9 @@ def _scanZarrGroup(self, group, results=None): """ if results is None: results = {'best': None, 'series': [], 'associated': []} + if isinstance(group, zarr.core.Array): + self._scanZarrArray(None, group, results) + return results for val in group.values(): if isinstance(val, zarr.core.Array): self._scanZarrArray(group, val, results) @@ -205,7 +208,7 @@ def _zarrFindLevels(self): baseGroup, baseArray = self._series[0] for idx, (_, arr) in enumerate(self._series): levels[idx][0] = arr - arrs = [[arr for _, arr in s.arrays()] for s, _ in self._series] + arrs = [[arr for _, arr in s.arrays()] if s is not None else [a] for s, a in self._series] for idx, arr in enumerate(arrs[0]): if any(idx >= len(sarrs) for sarrs in arrs[1:]): break @@ -296,6 +299,16 @@ def _validateZarr(self): stride *= len(self._series) self._framecount = stride + def _nonemptyLevelsList(self, frame=0): + """ + Return a list of one value per level where the value is None if the + level does not exist in the file and any other value if it does. + + :param frame: the frame number. + :returns: a list of levels length. + """ + return [True if l is not None else None for l in self._levels[0][::-1]] + def getNativeMagnification(self): """ Get the magnification at a particular level. @@ -397,27 +410,31 @@ def getTile(self, x, y, z, pilImageAllowed=False, numpyAllowed=False, **kwargs): x1 //= scale y1 //= scale step //= scale - idx = [slice(None) for _ in arr.shape] - idx[self._axes['x']] = slice(x0, x1, step) - idx[self._axes['y']] = slice(y0, y1, step) - for key in self._axes: - if key in self._strides: - pos = (frame // self._strides[key]) % self._axisCounts[key] - idx[self._axes[key]] = slice(pos, pos + 1) - trans = [idx for idx in range(len(arr.shape)) - if idx not in {self._axes['x'], self._axes['y'], - self._axes.get('s', self._axes['x'])}] - squeezeCount = len(trans) - trans += [self._axes['y'], self._axes['x']] - if 's' in self._axes: - trans.append(self._axes['s']) - with self._tileLock: - tile = arr[tuple(idx)] - tile = np.transpose(tile, trans) - for _ in range(squeezeCount): - tile = tile.squeeze(0) - if len(tile.shape) == 2: - tile = np.expand_dims(tile, axis=2) + if step > 2 ** self._maxSkippedLevels: + tile = self._getTileFromEmptyLevel(x, y, z, **kwargs) + tile = large_image.tilesource.base._imageToNumpy(tile)[0] + else: + idx = [slice(None) for _ in arr.shape] + idx[self._axes['x']] = slice(x0, x1, step) + idx[self._axes['y']] = slice(y0, y1, step) + for key in self._axes: + if key in self._strides: + pos = (frame // self._strides[key]) % self._axisCounts[key] + idx[self._axes[key]] = slice(pos, pos + 1) + trans = [idx for idx in range(len(arr.shape)) + if idx not in {self._axes['x'], self._axes['y'], + self._axes.get('s', self._axes['x'])}] + squeezeCount = len(trans) + trans += [self._axes['y'], self._axes['x']] + if 's' in self._axes: + trans.append(self._axes['s']) + with self._tileLock: + tile = arr[tuple(idx)] + tile = np.transpose(tile, trans) + for _ in range(squeezeCount): + tile = tile.squeeze(0) + if len(tile.shape) == 2: + tile = np.expand_dims(tile, axis=2) return self._outputTile(tile, TILE_FORMAT_NUMPY, x, y, z, pilImageAllowed, numpyAllowed, **kwargs) diff --git a/test/datastore.py b/test/datastore.py index 177a8f9f5..0f0f4ab41 100644 --- a/test/datastore.py +++ b/test/datastore.py @@ -104,6 +104,8 @@ 'synthetic_multiaxis.zarr.db': 'sha512:2ca118b67ca73bbc6fe9542c5b71ee6cb5f45f5049575a4682290cec4cfb4deef29aee5e19fb5d4005167322668a94191a86f98f1125c94f5eef3e14c6ec6e26', # noqa # The same as above, but as a multi directory zip 'synthetic_multiaxis.zarr.zip': 'sha512:95da53061bd09deaf4357e745404780d78a0949935f82c10ee75237e775345caace18fad3f05c3452ba36efca6b3ed58d815d041f33197497ab53d2c80b9e2ac', # noqa + # Single flat array zarr + 'flat2.zarr.zip': 'sha512:c49ff5fbfa73615da4c2a7c8602723297d604892b848860a068ab200245eec6c4f638f35d0b40cde0233c55faa6dc4e46351a841b481211f36dc5fb43765d818', # noqa }