Speed up opening some multi source files

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Improvements
 - Speed up rendering Girder item page metadata in some instances ([#1031](../../pull/1031))
+- Speed up opening some multi source files ([#1033](../../pull/1033))
 
 ### Bug Fixes
 - Fix an issue with non-square tiles in the multi source ([#1032](../../pull/1032))

sources/multi/large_image_source_multi/__init__.py

@@ -701,12 +701,10 @@ def _frameDictToFrames(self, frameDict):
                 frames.append(frame)
         return frames
 
-    def _collectFrames(self, checkAll=False):
+    def _collectFrames(self):
         """
         Using the specification in _info, enumerate the source files and open
         at least the first two of them to build up the frame specifications.
-
-        :param checkAll: if True, open all source files.
         """
         self._sources = sources = self._resolveFramePaths(self._info['sources'])
         self.logger.debug('Sources: %r', sources)
@@ -730,11 +728,16 @@ def _collectFrames(self, checkAll=False):
         # Walk through the sources, opening at least the first two, and
         # construct a frame list.  Each frame is a list of sources that affect
         # it along with the frame number from that source.
+        lastSource = None
         for sourceIdx, source in enumerate(sources):
             path = source['path']
             if os.path.abspath(path) == absLargeImagePath:
                 raise TileSourceError('Multi source specification is self-referential')
-            if numChecked < 2 or checkAll or not self._info.get('uniformSources'):
+            similar = False
+            if (lastSource and source['path'] == lastSource['path'] and
+                    source.get('params') == lastSource.get('params')):
+                similar = True
+            if not similar and (numChecked < 2 or not self._info.get('uniformSources')):
                 # need kwargs of frame, style?
                 ts = self._openSource(source)
                 self.tileWidth = self.tileWidth or ts.tileWidth
@@ -750,6 +753,7 @@ def _collectFrames(self, checkAll=False):
                     if not hasattr(self, '_bands'):
                         self._bands = {}
                     self._bands.update(tsMeta['bands'])
+                lastSource = source
             bbox = self._sourceBoundingBox(source, tsMeta['sizeX'], tsMeta['sizeY'])
             computedWidth = max(computedWidth, int(math.ceil(bbox['right'])))
             computedHeight = max(computedHeight, int(math.ceil(bbox['bottom'])))
@@ -809,7 +813,7 @@ def _openSource(self, source, params=None):
             openFunc = large_image.tilesource.AvailableTileSources[source['sourceName']]
         if params is None:
             params = source.get('params', {})
-        return openFunc(source['path'], **params, format=format)
+        return openFunc(source['path'], **params)
 
     def getAssociatedImage(self, imageKey, *args, **kwargs):
         """

test/datastore.py

@@ -87,6 +87,8 @@
     # keeping two levels
     'level-0-frames-0-320.dcm': 'sha512:c3c39e133988f29a99d87107f3b8fbef1c6f530350a9192671f237862731d6f44d18965773a499867d853cbf22aaed9ea1670ce0defda125efe6a8c0cc63c316',  # noqa
     'level-1-frames-0-20.dcm': 'sha512:cc414f0ec2f6ea0d41fa7677e5ce58d72b7541c21dd5c3a0106bf2d1814903daaeba61ae3c3cc3c46ed86210f04c7ed5cff0fc76c7305765f82642ad7ed4caa7',  # noqa
+    # Composite XY frames using the multi source
+    'multi-source-composite.yaml': 'sha512:61b79d43592d68509ec8a3e10b18689ac896c5753e8c1e5be349ff857ac981f8e980c4ad15bab84a3d67b08ada956d195557c64e2fd116dcdf9dc054f679573d',  # noqa
 }
 
 

test/test_source_base.py

@@ -48,7 +48,7 @@
     },
     'multi': {
         'read': r'\.(yml|yaml)$',
-        'skip': r'(multi_source\.yml)$',
+        'skip': r'(multi_source\.yml|multi-source-composite\.yaml)$',
     },
     'nd2': {
         'read': r'\.(nd2)$',

test/test_source_multi.py

@@ -191,3 +191,21 @@ def testFramesAsAxes():
     assert 'IndexZ' in tileMetadata['frames'][0]
     assert tileMetadata['IndexRange']['IndexC'] == 7
     assert tileMetadata['IndexRange']['IndexZ'] == 8
+
+
+@pytest.mark.skipif(sys.version_info < (3, 7), reason='requires python >= 3.7 for a sub-source')
+@pytest.mark.skipif(sys.version_info >= (3, 11), reason='requires python3.11 wheels')
+def testMultiComposite():
+    datastore.fetch('ITGA3Hi_export_crop2.nd2')
+    imagePath = datastore.fetch('multi-source-composite.yaml')
+
+    source = large_image_source_multi.open(imagePath)
+    tileMetadata = source.getMetadata()
+    assert tileMetadata['tileWidth'] == 1024
+    assert tileMetadata['tileHeight'] == 1022
+    assert tileMetadata['sizeX'] == 25906
+    assert tileMetadata['sizeY'] == 19275
+    assert tileMetadata['levels'] == 6
+    assert len(tileMetadata['frames']) == 116
+
+    utilities.checkTilesZXY(source, tileMetadata)