From 05e0f1d509998d1a0ad9f2f1766cb736073297f1 Mon Sep 17 00:00:00 2001 From: David Manthey Date: Tue, 10 Oct 2023 15:11:44 -0400 Subject: [PATCH] Add a Zarr tile source. This mostly reads omezarr (ome-ngff). --- .circleci/make_wheels.sh | 2 + .circleci/release_pypi.sh | 6 + CHANGELOG.md | 5 +- README.rst | 2 + docs/index.rst | 1 + docs/make_docs.sh | 1 + requirements-dev.txt | 1 + requirements-test-core.txt | 1 + requirements-test.txt | 1 + setup.py | 3 +- .../large_image_source_bioformats/__init__.py | 2 +- .../zarr/large_image_source_zarr/__init__.py | 444 ++++++++++++++++++ .../large_image_source_zarr/girder_source.py | 14 + sources/zarr/setup.py | 76 +++ test/datastore.py | 6 + test/test_source_base.py | 5 +- 16 files changed, 565 insertions(+), 5 deletions(-) create mode 100644 sources/zarr/large_image_source_zarr/__init__.py create mode 100644 sources/zarr/large_image_source_zarr/girder_source.py create mode 100644 sources/zarr/setup.py diff --git a/.circleci/make_wheels.sh b/.circleci/make_wheels.sh index 7444c0ed8..ce9251cda 100755 --- a/.circleci/make_wheels.sh +++ b/.circleci/make_wheels.sh @@ -55,3 +55,5 @@ cd "$ROOTPATH/sources/tifffile" pip wheel . --no-deps -w ~/wheels && rm -rf build cd "$ROOTPATH/sources/vips" pip wheel . --no-deps -w ~/wheels && rm -rf build +cd "$ROOTPATH/sources/zarr" +pip wheel . --no-deps -w ~/wheels && rm -rf build diff --git a/.circleci/release_pypi.sh b/.circleci/release_pypi.sh index f02dc0a3c..0fa778535 100755 --- a/.circleci/release_pypi.sh +++ b/.circleci/release_pypi.sh @@ -136,3 +136,9 @@ cp "$ROOTPATH/LICENSE" . python setup.py sdist pip wheel . --no-deps -w dist twine upload --verbose dist/* +cd "$ROOTPATH/sources/zarr" +cp "$ROOTPATH/README.rst" . +cp "$ROOTPATH/LICENSE" . +python setup.py sdist +pip wheel . --no-deps -w dist +twine upload --verbose dist/* diff --git a/CHANGELOG.md b/CHANGELOG.md index 74c131f78..2bc7ba381 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ # Change Log -## 1.25.1 +## 1.26.0 + +### Features +- Add a Zarr tile source (including ome.zarr) ([#1350](../../pull/1350)) ### Improvements - Style can specify a dtype of 'source' to maintain the original dtype even when compositing frames ([#1326](../../pull/1326)) diff --git a/README.rst b/README.rst index ff6b10269..80a7ea3ff 100644 --- a/README.rst +++ b/README.rst @@ -142,6 +142,8 @@ Large Image consists of several Python modules designed to work together. These - ``large-image-source-vips``: A tile source for reading any files handled by libvips. This also can be used for writing tiled images from numpy arrays. + - ``large-image-source-zarr``: A tile source using the zarr library that can handle OME-Zarr (OME-NGFF) files as well as some other zarr files. + - ``large-image-source-test``: A tile source that generates test tiles, including a simple fractal pattern. Useful for testing extreme zoom levels. - ``large-image-source-dummy``: A tile source that does nothing. diff --git a/docs/index.rst b/docs/index.rst index 99e168354..5846025a2 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -34,6 +34,7 @@ _build/large_image_source_tiff/modules _build/large_image_source_tifffile/modules _build/large_image_source_vips/modules + _build/large_image_source_zarr/modules _build/large_image_converter/modules _build/large_image_tasks/modules _build/girder_large_image/modules diff --git a/docs/make_docs.sh b/docs/make_docs.sh index 4d0fff5d0..ff12d3dfc 100755 --- a/docs/make_docs.sh +++ b/docs/make_docs.sh @@ -35,6 +35,7 @@ sphinx-apidoc -f -o _build/large_image_source_test ../sources/test/large_image_s sphinx-apidoc -f -o _build/large_image_source_tiff ../sources/tiff/large_image_source_tiff sphinx-apidoc -f -o _build/large_image_source_tifffile ../sources/tifffile/large_image_source_tifffile sphinx-apidoc -f -o _build/large_image_source_vips ../sources/vips/large_image_source_vips +sphinx-apidoc -f -o _build/large_image_source_zarr ../sources/zarr/large_image_source_zarr sphinx-apidoc -f -o _build/large_image_converter ../utilities/converter/large_image_converter sphinx-apidoc -f -o _build/large_image_tasks ../utilities/tasks/large_image_tasks sphinx-apidoc -f -o _build/girder_large_image ../girder/girder_large_image diff --git a/requirements-dev.txt b/requirements-dev.txt index 4e7e02d2f..2ecb49a80 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -16,6 +16,7 @@ girder-jobs>=3.0.3 -e sources/tiff -e sources/tifffile -e sources/vips +-e sources/zarr # must be after sources/tiff -e sources/ometiff # must be after source/gdal diff --git a/requirements-test-core.txt b/requirements-test-core.txt index bf1641ca1..61dce4429 100644 --- a/requirements-test-core.txt +++ b/requirements-test-core.txt @@ -14,6 +14,7 @@ sources/test sources/tiff sources/tifffile ; python_version >= '3.7' sources/vips +sources/zarr # must be after sources/tiff sources/ometiff # must be after source/gdal diff --git a/requirements-test.txt b/requirements-test.txt index 36d016fc4..ff21f2fc6 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -16,6 +16,7 @@ sources/test sources/tiff sources/tifffile ; python_version >= '3.7' sources/vips +sources/zarr # must be after sources/tiff sources/ometiff # must be after source/gdal diff --git a/setup.py b/setup.py index 635945a04..3ee7ba9f5 100644 --- a/setup.py +++ b/setup.py @@ -63,6 +63,7 @@ def prerelease_local_scheme(version): 'tiff': [f'large-image-source-tiff{limit_version}'], 'tifffile': [f'large-image-source-tifffile{limit_version} ; python_version >= "3.7"'], 'vips': [f'large-image-source-vips{limit_version}'], + 'zarr': [f'large-image-source-zarr{limit_version}'], } extraReqs.update(sources) extraReqs['sources'] = list(set(itertools.chain.from_iterable(sources.values()))) @@ -74,7 +75,7 @@ def prerelease_local_scheme(version): # from pypi with all needed dependencies. extraReqs['common'] = list(set(itertools.chain.from_iterable(extraReqs[key] for key in { 'memcached', 'colormaps', 'performance', - 'deepzoom', 'dicom', 'multi', 'nd2', 'test', 'tifffile', + 'deepzoom', 'dicom', 'multi', 'nd2', 'test', 'tifffile', 'zarr', })) | { f'large-image-source-pil[all]{limit_version}', f'large-image-source-rasterio[all]{limit_version} ; python_version >= "3.8"', diff --git a/sources/bioformats/large_image_source_bioformats/__init__.py b/sources/bioformats/large_image_source_bioformats/__init__.py index 0783c9dbd..2be2f8894 100644 --- a/sources/bioformats/large_image_source_bioformats/__init__.py +++ b/sources/bioformats/large_image_source_bioformats/__init__.py @@ -62,7 +62,7 @@ # Default to ignoring files with no extension and some specific extensions. config.ConfigValues['source_bioformats_ignored_names'] = \ - r'(^[^.]*|\.(jpg|jpeg|jpe|png|tif|tiff|ndpi|nd2|ome|nc|json|isyntax|mrxs))$' + r'(^[^.]*|\.(jpg|jpeg|jpe|png|tif|tiff|ndpi|nd2|ome|nc|json|isyntax|mrxs|zarr(\.db|\.zip)))$' def _monitor_thread(): diff --git a/sources/zarr/large_image_source_zarr/__init__.py b/sources/zarr/large_image_source_zarr/__init__.py new file mode 100644 index 000000000..e6eddfc82 --- /dev/null +++ b/sources/zarr/large_image_source_zarr/__init__.py @@ -0,0 +1,444 @@ +import functools +import math +import operator +import os +import threading + +import numpy as np +import packaging.version +import zarr + +import large_image +from large_image.cache_util import LruCacheMetaclass, methodcache +from large_image.constants import TILE_FORMAT_NUMPY, SourcePriority +from large_image.exceptions import TileSourceError, TileSourceFileNotFoundError +from large_image.tilesource import FileTileSource +from large_image.tilesource.utilities import nearPowerOfTwo + +try: + from importlib.metadata import PackageNotFoundError + from importlib.metadata import version as _importlib_version +except ImportError: + from importlib_metadata import PackageNotFoundError + from importlib_metadata import version as _importlib_version +try: + __version__ = _importlib_version(__name__) +except PackageNotFoundError: + # package is not installed + pass + + +class ZarrFileTileSource(FileTileSource, metaclass=LruCacheMetaclass): + """ + Provides tile access to files that the zarr library can read. + """ + + cacheName = 'tilesource' + name = 'zarr' + extensions = { + None: SourcePriority.LOW, + 'zarr': SourcePriority.PREFERRED, + 'zgroup': SourcePriority.PREFERRED, + 'zattrs': SourcePriority.PREFERRED, + 'db': SourcePriority.MEDIUM, + } + + _tileSize = 512 + _minTileSize = 128 + _maxTileSize = 1024 + _minAssociatedImageSize = 64 + _maxAssociatedImageSize = 8192 + + def __init__(self, path, **kwargs): + """ + Initialize the tile class. See the base class for other available + parameters. + + :param path: a filesystem path for the tile source. + """ + super().__init__(path, **kwargs) + + self._largeImagePath = str(self._getLargeImagePath()) + self._zarr = None + if not os.path.isfile(self._largeImagePath) and '//:' not in self._largeImagePath: + raise TileSourceFileNotFoundError(self._largeImagePath) from None + try: + self._zarr = zarr.open(zarr.SQLiteStore(self._largeImagePath)) + except Exception: + try: + self._zarr = zarr.open(self._largeImagePath) + except Exception: + if os.path.basename(self._largeImagePath) in {'.zgroup', '.zattrs'}: + try: + self._zarr = zarr.open(os.path.dirname(self._largeImagePath)) + except Exception: + pass + if self._zarr is None: + if not os.path.isfile(self._largeImagePath): + raise TileSourceFileNotFoundError(self._largeImagePath) from None + msg = 'File cannot be opened via zarr.' + raise TileSourceError(msg) + try: + self._validateZarr() + except TileSourceError: + raise + except Exception: + msg = 'File cannot be opened -- not an OME NGFF file or understandable zarr file.' + raise TileSourceError(msg) + self._tileLock = threading.RLock() + + def _getGeneralAxes(self, arr): + """ + Examine a zarr array an guess what the axes are. We assume the two + maximal dimensions are y, x. Then, if there is a dimension that is 3 + or 4 in length, it is channels. If there is more than one other that + is not 1, we don't know how it is sorted, so we will fail. + + :param arr: a zarr array. + :return: a dictionary of axes with the axis as the key and the index + within the array axes as the value. + """ + shape = arr.shape + maxIndex = shape.index(max(shape)) + secondMaxIndex = shape.index(max(x for idx, x in enumerate(shape) if idx != maxIndex)) + axes = { + 'x': max(maxIndex, secondMaxIndex), + 'y': min(maxIndex, secondMaxIndex), + } + for idx, val in enumerate(shape): + if idx not in axes.values() and val == 4 and 'c' not in axes: + axes['c'] = idx + if idx not in axes.values() and val == 3: + axes['c'] = idx + for idx, val in enumerate(shape): + if idx not in axes.values() and val > 1: + if 'f' in axes: + msg = 'Too many large axes' + raise TileSourceError(msg) + axes['f'] = idx + return axes + + def _scanZarrArray(self, group, arr, results): + """ + Scan a zarr array and determine if is the maximal dimension array we + can read. If so, update a results dictionary with the information. If + it is the shape as a previous maximum, append it as a multi-series set. + If not, and it is small enough, add it to a list of possible associated + images. + + :param group: a zarr group; the parent of the array. + :param arr: a zarr array. + :param results: a dictionary to store the results. 'best' contains a + tuple that is used to find the maximum size array, preferring ome + arrays, then total pixels, then channels. 'is_ome' is a boolean. + 'series' is a list of the found groups and arrays that match the + best criteria. 'axes' and 'channels' are from the best array. + 'associated' is a list of all groups and arrays that might be + associated images. These have to be culled for the actual groups + used in the series. + """ + attrs = group.attrs.asdict() + min_version = packaging.version.Version('0.4') + is_ome = ( + isinstance(attrs['multiscales'], list) and + 'omero' in attrs and + isinstance(attrs['omero'], dict) and + all(isinstance(m, dict) for m in attrs['multiscales']) and + all(packaging.version.Version(m['version']) >= min_version + for m in attrs['multiscales'] if 'version' in m)) + channels = None + if is_ome: + axes = {axis['name']: idx for idx, axis in enumerate( + attrs['multiscales'][0]['axes'])} + if isinstance(attrs['omero'].get('channels'), list): + channels = [channel['label'] for channel in attrs['omero']['channels']] + if all(channel.startswith('Channel ') for channel in channels): + channels = None + else: + try: + axes = self._getGeneralAxes(arr) + except TileSourceError: + return + if 'x' not in axes or 'y' not in axes: + return + # Change this to `math.prod(arr.shape)` when dropping Python 3.7 + check = (is_ome, functools.reduce(operator.mul, arr.shape, 1), + channels is not None, tuple(axes.keys()), + tuple(channels) if channels else ()) + if results['best'] is None or check > results['best']: + results['best'] = check + results['series'] = [(group, arr)] + results['is_ome'] = is_ome + results['axes'] = axes + results['channels'] = channels + elif check == results['best']: + results['series'].append((group, arr)) + if not any(group is g for g, _ in results['associated']): + axes = {k: v for k, v in axes.items() if arr.shape[axes[k]] > 1} + if (len(axes) <= 3 and + self._minAssociatedImageSize <= arr.shape[axes['x']] <= + self._maxAssociatedImageSize and + self._minAssociatedImageSize <= arr.shape[axes['y']] <= + self._maxAssociatedImageSize and + (len(axes) == 2 or ('c' in axes and arr.shape[axes['c']] in {1, 3, 4}))): + results['associated'].append((group, arr)) + + def _scanZarrGroup(self, group, results=None): + """ + Scan a zarr group for usable arrays. + + :param group: a zarr group + :param results: a results dicitionary, updated. + :returns: the results dictionary. + """ + if results is None: + results = {'best': None, 'series': [], 'associated': []} + for val in group.values(): + if isinstance(val, zarr.core.Array): + self._scanZarrArray(group, val, results) + elif isinstance(val, zarr.hierarchy.Group): + results = self._scanZarrGroup(val, results) + return results + + def _zarrFindLevels(self): + """ + Find usable multi-level images. This checks that arrays are nearly a + power of two. This updates self._levels and self._populatedLevels. + self._levels is an array the same length as the number of series, each + entry of which is an array of the number of conceptual tile levels + where each entry of that is either the zarr array that can be used to + get pixels or None if it is not populated. + """ + levels = [[None] * self.levels for _ in self._series] + baseGroup, baseArray = self._series[0] + for idx, (_, arr) in enumerate(self._series): + levels[idx][0] = arr + arrs = [[arr for _, arr in s.arrays()] for s, _ in self._series] + for idx, arr in enumerate(arrs[0]): + if any(idx >= len(sarrs) for sarrs in arrs[1:]): + break + if any(arr.shape != sarrs[idx].shape for sarrs in arrs[1:]): + continue + if (nearPowerOfTwo(self.sizeX, arr.shape[self._axes['x']]) and + nearPowerOfTwo(self.sizeY, arr.shape[self._axes['y']])): + level = int(round(math.log(self.sizeX / arr.shape[self._axes['x']]) / math.log(2))) + if level < self.levels and levels[0][level] is None: + for sidx in range(len(self._series)): + levels[sidx][level] = arrs[sidx][idx] + self._levels = levels + self._populatedLevels = len([l for l in self._levels[0] if l is not None]) + # TODO: check for inefficient file and raise warning + + def _getScale(self): + """ + Get the scale values from the ome metadata and populate the class + values for _mm_x and _mm_y. + """ + unit = {'micrometer': 1e-3, 'millimeter': 1, 'meter': 1e3} + self._mm_x = self._mm_y = None + baseGroup, baseArray = self._series[0] + try: + ms = baseGroup.attrs.asdict()['multiscales'][0] + self._mm_x = ms['datasets'][0]['coordinateTransformations'][0][ + 'scale'][self._axes['x']] * unit[ms['axes'][self._axes['x']]['unit']] + self._mm_y = ms['datasets'][0]['coordinateTransformations'][0][ + 'scale'][self._axes['y']] * unit[ms['axes'][self._axes['y']]['unit']] + except Exception: + pass + + def _validateZarr(self): + """ + Validate that we can read tiles from the zarr parent group in + self._zarr. Set up the appropriate class variables. + """ + found = self._scanZarrGroup(self._zarr) + if found['best'] is None: + msg = 'No data array that can be used.' + raise TileSourceError(msg) + self._series = found['series'] + baseGroup, baseArray = self._series[0] + self._is_ome = found['is_ome'] + self._axes = {k.lower(): v for k, v in found['axes'].items() if baseArray.shape[v] > 1} + if len(self._series) > 1 and 'xy' in self._axes: + msg = 'Conflicting xy axis data.' + raise TileSourceError(msg) + self._channels = found['channels'] + self._associatedImages = [ + (g, a) for g, a in found['associated'] if not any(g is gb for gb, _ in self._series)] + self.sizeX = baseArray.shape[self._axes['x']] + self.sizeY = baseArray.shape[self._axes['y']] + self.tileWidth = ( + baseArray.chunks[self._axes['x']] + if self._minTileSize <= baseArray.chunks[self._axes['x']] <= self._maxTileSize else + self._tileSize) + self.tileHeight = ( + baseArray.chunks[self._axes['y']] + if self._minTileSize <= baseArray.chunks[self._axes['y']] <= self._maxTileSize else + self._tileSize) + # If we wanted to require equal tile width and height: + # self.tileWidth = self.tileHeight = self._tileSize + # if (baseArray.chunks[self._axes['x']] == baseArray.chunks[self._axes['y']] and + # self._minTileSize <= baseArray.chunks[self._axes['x']] <= self._maxTileSize): + # self.tileWidth = self.tileHeight = baseArray.chunks[self._axes['x']] + self.levels = int(max(1, math.ceil(math.log(max( + self.sizeX / self.tileWidth, self.sizeY / self.tileHeight)) / math.log(2)) + 1)) + self._dtype = baseArray.dtype + self._bandCount = 1 + if ('c' in self._axes and 's' not in self._axes and not self._channels and + baseArray.shape[self._axes.get('c')] in {1, 3, 4}): + self._bandCount = baseArray.shape[self._axes['c']] + self._axes['s'] = self._axes.pop('c') + self._zarrFindLevels() + self._getScale() + stride = 1 + self._strides = {} + self._axisCounts = {} + for _, k in sorted((-'tzc'.index(k) if k in 'tzc' else 1, k) + for k in self._axes if k not in 'xys'): + self._strides[k] = stride + self._axisCounts[k] = baseArray.shape[self._axes[k]] + stride *= baseArray.shape[self._axes[k]] + if len(self._series) > 1: + self._strides['xy'] = stride + self._axisCounts['xy'] = len(self._series) + stride *= len(self._series) + self._framecount = stride + + def getNativeMagnification(self): + """ + Get the magnification at a particular level. + + :return: magnification, width of a pixel in mm, height of a pixel in mm. + """ + mm_x = self._mm_x + mm_y = self._mm_y + # Estimate the magnification; we don't have a direct value + mag = 0.01 / mm_x if mm_x else None + return { + 'magnification': getattr(self, '_magnification', mag), + 'mm_x': mm_x, + 'mm_y': mm_y, + } + + def getMetadata(self): + """ + Return a dictionary of metadata containing levels, sizeX, sizeY, + tileWidth, tileHeight, magnification, mm_x, mm_y, and frames. + + :returns: metadata dictionary. + """ + result = super().getMetadata() + if self._framecount > 1: + result['frames'] = frames = [] + for idx in range(self._framecount): + frame = {'Frame': idx} + for axis in self._strides: + frame['Index' + axis.upper()] = ( + idx // self._strides[axis]) % self._axisCounts[axis] + frames.append(frame) + self._addMetadataFrameInformation(result, getattr(self, '_channels', None)) + return result + + def getInternalMetadata(self, **kwargs): + """ + Return additional known metadata about the tile source. Data returned + from this method is not guaranteed to be in any particular format or + have specific values. + + :returns: a dictionary of data or None. + """ + result = {} + result['zarr'] = { + 'base': self._zarr.attrs.asdict(), + 'main': self._series[0][0].attrs.asdict(), + } + return result + + def getAssociatedImagesList(self): + """ + Get a list of all associated images. + + :return: the list of image keys. + """ + return [f'image_{idx}' for idx in range(len(self._associatedImages))] + + def _getAssociatedImage(self, imageKey): + """ + Get an associated image in PIL format. + + :param imageKey: the key of the associated image. + :return: the image in PIL format or None. + """ + if not imageKey.startswith('image_'): + return + try: + idx = int(imageKey[6:]) + except Exception: + return + if idx < 0 or idx >= len(self._associatedImages): + return + group, arr = self._associatedImages[idx] + axes = self._getGeneralAxes(arr) + trans = [idx for idx in range(len(arr.shape)) + if idx not in axes.values()] + [axes['y'], axes['x']] + if 'c' in axes or 's' in axes: + trans.append(axes.get('c', axes.get('s'))) + with self._tileLock: + img = np.transpose(arr, trans).squeeze() + if len(img.shape) == 2: + img.expand_dims(axis=2) + return large_image.tilesource.base._imageToPIL(img) + + @methodcache() + def getTile(self, x, y, z, pilImageAllowed=False, numpyAllowed=False, **kwargs): + frame = self._getFrame(**kwargs) + self._xyzInRange(x, y, z, frame, self._framecount) + x0, y0, x1, y1, step = self._xyzToCorners(x, y, z) + sidx = 0 if len(self._series) <= 1 else frame // self._strides['xy'] + targlevel = self.levels - 1 - z + while targlevel and self._levels[sidx][targlevel] is None: + targlevel -= 1 + arr = self._levels[sidx][targlevel] + scale = int(2 ** targlevel) + x0 //= scale + y0 //= scale + x1 //= scale + y1 //= scale + step //= scale + idx = [slice(None) for _ in arr.shape] + idx[self._axes['x']] = slice(x0, x1, step) + idx[self._axes['y']] = slice(y0, y1, step) + for key in self._axes: + if key in self._strides: + pos = (frame // self._strides[key]) % self._axisCounts[key] + idx[self._axes[key]] = slice(pos, pos + 1) + trans = [idx for idx in range(len(arr.shape)) + if idx not in {self._axes['x'], self._axes['y'], + self._axes.get('s', self._axes['x'])}] + squeezeCount = len(trans) + trans += [self._axes['y'], self._axes['x']] + if 's' in self._axes: + trans.append(self._axes['s']) + with self._tileLock: + tile = arr[tuple(idx)] + tile = np.transpose(tile, trans) + for _ in range(squeezeCount): + tile = tile.squeeze(0) + if len(tile.shape) == 2: + tile = np.expand_dims(tile, axis=2) + return self._outputTile(tile, TILE_FORMAT_NUMPY, x, y, z, + pilImageAllowed, numpyAllowed, **kwargs) + + +def open(*args, **kwargs): + """ + Create an instance of the module class. + """ + return ZarrFileTileSource(*args, **kwargs) + + +def canRead(*args, **kwargs): + """ + Check if an input can be read by the module class. + """ + return ZarrFileTileSource.canRead(*args, **kwargs) diff --git a/sources/zarr/large_image_source_zarr/girder_source.py b/sources/zarr/large_image_source_zarr/girder_source.py new file mode 100644 index 000000000..1ae2b592e --- /dev/null +++ b/sources/zarr/large_image_source_zarr/girder_source.py @@ -0,0 +1,14 @@ +from girder_large_image.girder_tilesource import GirderTileSource + +from . import ZarrFileTileSource + + +class ZarrGirderTileSource(ZarrFileTileSource, GirderTileSource): + """ + Provides tile access to Girder items with files that OME Zarr can read. + """ + + cacheName = 'tilesource' + name = 'zarr' + + _mayHaveAdjacentFiles = True diff --git a/sources/zarr/setup.py b/sources/zarr/setup.py new file mode 100644 index 000000000..f243738ed --- /dev/null +++ b/sources/zarr/setup.py @@ -0,0 +1,76 @@ +import os + +from setuptools import find_packages, setup + +description = 'A OME Zarr tilesource for large_image.' +long_description = description + '\n\nSee the large-image package for more details.' + + +def prerelease_local_scheme(version): + """ + Return local scheme version unless building on master in CircleCI. + + This function returns the local scheme version number + (e.g. 0.0.0.dev+g) unless building on CircleCI for a + pre-release in which case it ignores the hash and produces a + PEP440 compliant pre-release version number (e.g. 0.0.0.dev). + """ + from setuptools_scm.version import get_local_node_and_date + + if os.getenv('CIRCLE_BRANCH') in ('master', ): + return '' + else: + return get_local_node_and_date(version) + + +try: + from setuptools_scm import get_version + + version = get_version(root='../..', local_scheme=prerelease_local_scheme) + limit_version = f'>={version}' if '+' not in version else '' +except (ImportError, LookupError): + limit_version = '' + +setup( + name='large-image-source-zarr', + use_scm_version={'root': '../..', 'local_scheme': prerelease_local_scheme, + 'fallback_version': '0.0.0'}, + setup_requires=['setuptools-scm'], + description=description, + long_description=long_description, + license='Apache Software License 2.0', + author='Kitware, Inc.', + author_email='kitware@kitware.com', + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + ], + install_requires=[ + f'large-image{limit_version}', + 'zarr ; python_version >= "3.8"', + 'zarr<2.11 ; python_version < "3.8"', + 'importlib-metadata<5 ; python_version < "3.8"', + ], + extras_require={ + 'girder': f'girder-large-image{limit_version}', + }, + keywords='large_image, tile source', + packages=find_packages(exclude=['test', 'test.*']), + url='https://github.com/girder/large_image', + python_requires='>=3.6', + entry_points={ + 'large_image.source': [ + 'zarr = large_image_source_zarr:ZarrFileTileSource', + ], + 'girder_large_image.source': [ + 'zarr = large_image_source_zarr.girder_source:ZarrGirderTileSource', + ], + }, +) diff --git a/test/datastore.py b/test/datastore.py index d36a36cff..177a8f9f5 100644 --- a/test/datastore.py +++ b/test/datastore.py @@ -98,6 +98,12 @@ 'synthetic_imagej.tiff': 'sha512:f4fcf9633b7fc8c819aace248352b3ef89364418d734aa8326096829d64956f11b961c24003bd5ca1f7e46462a8230b64177b37b1517a416d7c7123a0932d5dc', # noqa # Blank untiled tiff with bad tifffile axes information 'bad_axes_description.tif': 'sha512:cbe82614a83d315a2f22c32b14fd32560fadb3f263bfeffc75f42f88e535e8a11728f691499cb7f5714bbbc55938c472ee4a706d7281bdfefe6fdb965c0a2d6b', # noqa + # Converted from the blank qptiff file + 'synthetic_channels.zarr.db': 'sha512:84a1e5e55931ca17e914b97c3e90bba786ce0a53eec61956a0026a65eea759f45193052e70add531b9debd13597433fdc06fc0abfa5ebae628fa64d34e0d750d', # noqa + # Converted from a file called normmedia...nd2 and made mostly dull + 'synthetic_multiaxis.zarr.db': 'sha512:2ca118b67ca73bbc6fe9542c5b71ee6cb5f45f5049575a4682290cec4cfb4deef29aee5e19fb5d4005167322668a94191a86f98f1125c94f5eef3e14c6ec6e26', # noqa + # The same as above, but as a multi directory zip + 'synthetic_multiaxis.zarr.zip': 'sha512:95da53061bd09deaf4357e745404780d78a0949935f82c10ee75237e775345caace18fad3f05c3452ba36efca6b3ed58d815d041f33197497ab53d2c80b9e2ac', # noqa } diff --git a/test/test_source_base.py b/test/test_source_base.py index 288f6d4ac..e63fbf1b1 100644 --- a/test/test_source_base.py +++ b/test/test_source_base.py @@ -82,16 +82,17 @@ 'skipTiles': r'(sample_image\.ptif|one_layer_missing_tiles)'}, 'tifffile': { 'read': r'', - 'noread': r'((\.(nc|nd2|yml|yaml|json|czi|png|jpg|jpeg|jp2|dcm)$)' + + 'noread': r'((\.(nc|nd2|yml|yaml|json|czi|png|jpg|jpeg|jp2|dcm|zarr\.db|zarr\.zip)$)' + (r'|bad_axes' if sys.version_info < (3, 9) else '') + r')', 'python': sys.version_info >= (3, 7), }, 'vips': { 'read': r'', - 'noread': r'\.(nc|nd2|yml|yaml|json|czi|png|svs|scn)$', + 'noread': r'\.(nc|nd2|yml|yaml|json|czi|png|svs|scn|zarr\.db|zarr\.zip)$', 'skipTiles': r'(sample_image\.ptif|one_layer_missing_tiles|JK-kidney_B-gal_H3_4C_1-500sec\.jp2|extraoverview)' # noqa }, + 'zarr': {'read': r'\.(zarr|zgroup|zattrs|db|zarr\.zip)$'}, }