From dabe48d941b13353fb6268c276630224877c10ae Mon Sep 17 00:00:00 2001 From: David Manthey Date: Mon, 23 Jan 2023 09:34:43 -0500 Subject: [PATCH] Speed up opening some multi source files --- CHANGELOG.md | 1 + .../multi/large_image_source_multi/__init__.py | 14 +++++++++----- test/datastore.py | 2 ++ test/test_source_base.py | 2 +- test/test_source_multi.py | 18 ++++++++++++++++++ 5 files changed, 31 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b3067d07..c42ed47b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Improvements - Speed up rendering Girder item page metadata in some instances ([#1031](../../pull/1031)) +- Speed up opening some multi source files ([#1033](../../pull/1033)) ### Bug Fixes - Fix an issue with non-square tiles in the multi source ([#1032](../../pull/1032)) diff --git a/sources/multi/large_image_source_multi/__init__.py b/sources/multi/large_image_source_multi/__init__.py index 000fe5207..de70591e8 100644 --- a/sources/multi/large_image_source_multi/__init__.py +++ b/sources/multi/large_image_source_multi/__init__.py @@ -701,12 +701,10 @@ def _frameDictToFrames(self, frameDict): frames.append(frame) return frames - def _collectFrames(self, checkAll=False): + def _collectFrames(self): """ Using the specification in _info, enumerate the source files and open at least the first two of them to build up the frame specifications. - - :param checkAll: if True, open all source files. """ self._sources = sources = self._resolveFramePaths(self._info['sources']) self.logger.debug('Sources: %r', sources) @@ -730,11 +728,16 @@ def _collectFrames(self, checkAll=False): # Walk through the sources, opening at least the first two, and # construct a frame list. Each frame is a list of sources that affect # it along with the frame number from that source. + lastSource = None for sourceIdx, source in enumerate(sources): path = source['path'] if os.path.abspath(path) == absLargeImagePath: raise TileSourceError('Multi source specification is self-referential') - if numChecked < 2 or checkAll or not self._info.get('uniformSources'): + similar = False + if (lastSource and source['path'] == lastSource['path'] and + source.get('params') == lastSource.get('params')): + similar = True + if not similar and (numChecked < 2 or not self._info.get('uniformSources')): # need kwargs of frame, style? ts = self._openSource(source) self.tileWidth = self.tileWidth or ts.tileWidth @@ -750,6 +753,7 @@ def _collectFrames(self, checkAll=False): if not hasattr(self, '_bands'): self._bands = {} self._bands.update(tsMeta['bands']) + lastSource = source bbox = self._sourceBoundingBox(source, tsMeta['sizeX'], tsMeta['sizeY']) computedWidth = max(computedWidth, int(math.ceil(bbox['right']))) computedHeight = max(computedHeight, int(math.ceil(bbox['bottom']))) @@ -809,7 +813,7 @@ def _openSource(self, source, params=None): openFunc = large_image.tilesource.AvailableTileSources[source['sourceName']] if params is None: params = source.get('params', {}) - return openFunc(source['path'], **params, format=format) + return openFunc(source['path'], **params) def getAssociatedImage(self, imageKey, *args, **kwargs): """ diff --git a/test/datastore.py b/test/datastore.py index 9902a6a8d..ded552062 100644 --- a/test/datastore.py +++ b/test/datastore.py @@ -87,6 +87,8 @@ # keeping two levels 'level-0-frames-0-320.dcm': 'sha512:c3c39e133988f29a99d87107f3b8fbef1c6f530350a9192671f237862731d6f44d18965773a499867d853cbf22aaed9ea1670ce0defda125efe6a8c0cc63c316', # noqa 'level-1-frames-0-20.dcm': 'sha512:cc414f0ec2f6ea0d41fa7677e5ce58d72b7541c21dd5c3a0106bf2d1814903daaeba61ae3c3cc3c46ed86210f04c7ed5cff0fc76c7305765f82642ad7ed4caa7', # noqa + # Composite XY frames using the multi source + 'multi-source-composite.yaml': 'sha512:61b79d43592d68509ec8a3e10b18689ac896c5753e8c1e5be349ff857ac981f8e980c4ad15bab84a3d67b08ada956d195557c64e2fd116dcdf9dc054f679573d', # noqa } diff --git a/test/test_source_base.py b/test/test_source_base.py index dc1565bb7..35d7be859 100644 --- a/test/test_source_base.py +++ b/test/test_source_base.py @@ -48,7 +48,7 @@ }, 'multi': { 'read': r'\.(yml|yaml)$', - 'skip': r'(multi_source\.yml)$', + 'skip': r'(multi_source\.yml|multi-source-composite\.yaml)$', }, 'nd2': { 'read': r'\.(nd2)$', diff --git a/test/test_source_multi.py b/test/test_source_multi.py index bea5b3422..7dd3e5268 100644 --- a/test/test_source_multi.py +++ b/test/test_source_multi.py @@ -191,3 +191,21 @@ def testFramesAsAxes(): assert 'IndexZ' in tileMetadata['frames'][0] assert tileMetadata['IndexRange']['IndexC'] == 7 assert tileMetadata['IndexRange']['IndexZ'] == 8 + + +@pytest.mark.skipif(sys.version_info < (3, 7), reason='requires python >= 3.7 for a sub-source') +@pytest.mark.skipif(sys.version_info >= (3, 11), reason='requires python3.11 wheels') +def testMultiComposite(): + datastore.fetch('ITGA3Hi_export_crop2.nd2') + imagePath = datastore.fetch('multi-source-composite.yaml') + + source = large_image_source_multi.open(imagePath) + tileMetadata = source.getMetadata() + assert tileMetadata['tileWidth'] == 1024 + assert tileMetadata['tileHeight'] == 1022 + assert tileMetadata['sizeX'] == 25906 + assert tileMetadata['sizeY'] == 19275 + assert tileMetadata['levels'] == 6 + assert len(tileMetadata['frames']) == 116 + + utilities.checkTilesZXY(source, tileMetadata)