Merge pull request #1054 from girder/multi-axes

Allow arbitrary axes in the multi and test sources.

CHANGELOG.md

@@ -6,7 +6,9 @@
 - ICC color profile support ([#1037](../../pull/1037), [#1043](../../pull/1043), [#1046](../../pull/1046), [#1048](../../pull/1048), [#1052](../../pull/1052))
 
   Note: ICC color profile adjustment to sRGB is the new default.  This can be disabled with a config setting or on a per tile source basis.
-
+
+- Support arbitrary axes in the test and multi sources ([#1054](../../pull/1054))
+
 ### Improvements
 - Speed up generating tiles for some multi source files ([#1035](../../pull/1035), [#1047](../../pull/1047))
 - Render item lists faster ([#1036](../../pull/1036))

large_image/tilesource/base.py

@@ -1549,6 +1549,8 @@ def getMetadata(self):
                 :IndexZ: optional if unique.  A 0-based index for z values
                 :IndexXY: optional if unique.  A 0-based index for view (xy)
                     values
+                :Index<axis>: optional if unique.  A 0-based index for an
+                    arbitrary axis.
                 :Index: a 0-based index of non-channel unique sets.  If the
                     frames vary only by channel and are adjacent, they will
                     have the same index.
@@ -1557,6 +1559,8 @@ def getMetadata(self):
                 frames if greater than 1 (e.g., if an entry like IndexXY is not
                 present, then all frames either do not have that value or have
                 a value of 0).
+            :IndexStride: a dictionary of the spacing between frames where
+                unique axes values change.
             :channels: optional.  If known, a list of channel names
             :channelmap: optional.  If known, a dictionary of channel names
                 with their offset into the channel list.
@@ -1591,9 +1595,11 @@ def _addMetadataFrameInformation(self, metadata, channels=None):
         if 'frames' not in metadata:
             return
         maxref = {}
-        refkeys = {'IndexC', 'IndexZ', 'IndexXY', 'IndexT'}
+        refkeys = {'IndexC'}
         index = 0
         for idx, frame in enumerate(metadata['frames']):
+            refkeys |= {key for key in frame
+                        if key.startswith('Index') and len(key.split('Index', 1)[1])}
             for key in refkeys:
                 if key in frame and frame[key] + 1 > maxref.get(key, 0):
                     maxref[key] = frame[key] + 1

sources/multi/large_image_source_multi/__init__.py

@@ -340,6 +340,14 @@
                 'layout and size, assume all sources are so similar',
             'type': 'boolean',
         },
+        'axes': {
+            'description': 'A list of additional axes that will be parsed.  '
+                           'The default axes are z, t, xy, and c.  It is '
+                           'recommended that additional axes use terse names '
+                           'and avoid x, y, and s.',
+            'type': 'array',
+            'items': {'type': 'string'},
+        },
         'sources': {
             'type': 'array',
             'items': SourceEntrySchema
@@ -390,6 +398,8 @@ def __init__(self, path, **kwargs):
 
         self._largeImagePath = self._getLargeImagePath()
         self._lastOpenSourceLock = threading.RLock()
+        # 'c' must be first as channels are special because they can have names
+        self._axesList = ['c', 'z', 't', 'xy']
         if not os.path.isfile(self._largeImagePath):
             try:
                 possibleYaml = self._largeImagePath.split('multi://', 1)[-1]
@@ -416,6 +426,9 @@ def __init__(self, path, **kwargs):
             self._validator.validate(self._info)
             self._basePath = Path(self._largeImagePath).parent
         self._basePath /= Path(self._info.get('basePath', '.'))
+        for axis in self._info.get('axes', []):
+            if axis not in self._axesList:
+                self._axesList.append(axis)
         self._collectFrames()
 
     def _resolvePathPatterns(self, sources, source):
@@ -454,7 +467,7 @@ def _resolvePathPatterns(self, sources, source):
                 else:
                     subsource[k] = v
             subsource['path'] = entry
-            for axis in ['z', 't', 'xy', 'c']:
+            for axis in self._axesList:
                 stepKey = '%sStep' % axis
                 valuesKey = '%sValues' % axis
                 if stepKey in source:
@@ -595,7 +608,7 @@ def _adjustFramesAsAxes(self, frames, idx, framesAsAxes):
             self.logger.warning('framesAsAxes strides do not use all frames.')
             self._warnedAdjustFramesAsAxes = True
         frame = frames[idx].copy()
-        for axis in ['c', 'z', 't', 'xy']:
+        for axis in self._axesList:
             frame.pop('Index' + axis.upper(), None)
         for axis, stride in framesAsAxes.items():
             frame['Index' + axis.upper()] = (idx // stride) % axisRange[axis]
@@ -621,34 +634,31 @@ def _addSourceToFrames(self, tsMeta, source, sourceIdx, frameDict):
         if len(channels) > len(self._channels):
             self._channels += channels[len(self._channels):]
         if not any(key in source for key in {
-                'frame', 'c', 'z', 't', 'xy',
-                'frameValues', 'cValues', 'zValues', 'tValues', 'xyValues'}):
+                'frame', 'frameValues'} |
+                set(self._axesList) |
+                {f'{axis}Values' for axis in self._axesList}):
             source = source.copy()
             if len(frameDict['byFrame']):
                 source['frame'] = max(frameDict['byFrame'].keys()) + 1
             if len(frameDict['byAxes']):
-                source['z'] = max(aKey[1] for aKey in frameDict['byAxes']) + 1
+                source['z'] = max(
+                    aKey[self._axesList.index('z')] for aKey in frameDict['byAxes']) + 1
         for frameIdx, frame in enumerate(frames):
             if 'frames' in source and frameIdx not in source['frames']:
                 continue
             if source.get('framesAsAxes'):
                 frame = self._adjustFramesAsAxes(frames, frameIdx, source.get('framesAsAxes'))
             fKey = self._axisKey(source, frameIdx, 'frame')
             cIdx = frame.get('IndexC', 0)
-            zIdx = frame.get('IndexZ', 0)
-            tIdx = frame.get('IndexT', 0)
-            xyIdx = frame.get('IndexXY', 0)
-            aKey = (self._axisKey(source, cIdx, 'c'),
-                    self._axisKey(source, zIdx, 'z'),
-                    self._axisKey(source, tIdx, 't'),
-                    self._axisKey(source, xyIdx, 'xy'))
+            aKey = tuple(self._axisKey(source, frame.get(f'Index{axis.upper()}') or 0, axis)
+                         for axis in self._axesList)
             channel = channels[cIdx] if cIdx < len(channels) else None
             if channel and channel not in self._channels and (
                     'channel' in source or 'channels' in source):
                 self._channels.append(channel)
             if (channel and channel in self._channels and
                     'c' not in source and 'cValues' not in source):
-                aKey = (self._channels.index(channel), aKey[1], aKey[2], aKey[3])
+                aKey = tuple([self._channels.index(channel)] + list(aKey[1:]))
             kwargs = source.get('params', {}).copy()
             if 'style' in source:
                 kwargs['style'] = source['style']
@@ -661,7 +671,7 @@ def _addSourceToFrames(self, tsMeta, source, sourceIdx, frameDict):
                 'kwargs': kwargs,
             })
             frameDict['axesAllowed'] = (frameDict['axesAllowed'] and (
-                len(frames) <= 1 or 'IndexRange' in tsMeta)) or aKey != (0, 0, 0, 0)
+                len(frames) <= 1 or 'IndexRange' in tsMeta)) or aKey != tuple([0] * len(aKey))
             frameDict['byAxes'].setdefault(aKey, [])
             frameDict['byAxes'][aKey].append({
                 'sourcenum': sourceIdx,
@@ -684,22 +694,16 @@ def _frameDictToFrames(self, frameDict):
                 frame = {'sources': frameDict['byFrame'].get(frameIdx, [])}
                 frames.append(frame)
         else:
-            axesCount = [max(aKey[idx] for aKey in frameDict['byAxes']) + 1 for idx in range(4)]
-            for xy, t, z, c in itertools.product(
-                    range(axesCount[3]), range(axesCount[2]),
-                    range(axesCount[1]), range(axesCount[0])):
-                aKey = (c, z, t, xy)
+            axesCount = [max(aKey[idx] for aKey in frameDict['byAxes']) + 1
+                         for idx in range(len(self._axesList))]
+            for aKey in itertools.product(*[range(count) for count in axesCount][::-1]):
+                aKey = tuple(aKey[::-1])
                 frame = {
                     'sources': frameDict['byAxes'].get(aKey, []),
                 }
-                if axesCount[0] > 1:
-                    frame['IndexC'] = c
-                if axesCount[1] > 1:
-                    frame['IndexZ'] = z
-                if axesCount[2] > 1:
-                    frame['IndexT'] = t
-                if axesCount[3] > 1:
-                    frame['IndexXY'] = xy
+                for idx, axis in enumerate(self._axesList):
+                    if axesCount[idx] > 1:
+                        frame[f'Index{axis.upper()}'] = aKey[idx]
                 frames.append(frame)
         return frames
 

sources/test/large_image_source_test/__init__.py

@@ -72,7 +72,8 @@ def __init__(self, ignored_path=None, minLevel=0, maxLevel=9,
         :param fractal: if True, and the tile size is square and a power of
             two, draw a simple fractal on the tiles.
         :param frames: if present, this is either a single number for generic
-            frames, or comma-separated list of c,z,t,xy.
+            frames, a comma-separated list of c,z,t,xy, or a string of the
+            form '<axis>=<count>,<axis>=<count>,...'.
         :param monochrome: if True, return single channel tiles.
         :param bands: if present, a comma-separated list of band names.
             Defaults to red,green,blue.
@@ -104,17 +105,26 @@ def __init__(self, ignored_path=None, minLevel=0, maxLevel=9,
         self.levels = self.maxLevel + 1
         if frames:
             frameList = []
-            counts = [int(part) for part in str(frames).split(',')]
-            self._framesParts = len(counts)
-            for fidx in itertools.product(*(range(part) for part in counts[::-1])):
+            if '=' not in str(frames) and ',' not in str(frames):
+                self._axes = [('f', 'Index', int(frames))]
+            elif '=' not in str(frames):
+                self._axes = [
+                    (axis, f'Index{axis.upper()}', int(part))
+                    for axis, part in zip(['c', 'z', 't', 'xy'], frames.split(','))]
+            else:
+                self._axes = [
+                    (part.split('=', 1)[0],
+                     f'Index{part.split("=", 1)[0].upper()}',
+                     int(part.split('=', 1)[1])) for part in frames.split(',')]
+            self._framesParts = len(self._axes)
+            axes = self._axes[::-1]
+            for fidx in itertools.product(*(range(part[-1]) for part in axes)):
                 curframe = {}
-                if len(fidx) > 1:
-                    for idx, (k, v) in enumerate(zip([
-                            'IndexC', 'IndexZ', 'IndexT', 'IndexXY'], list(fidx)[::-1])):
-                        if counts[idx] > 1:
-                            curframe[k] = v
-                else:
-                    curframe['Index'] = fidx[0]
+                for idx in range(len(fidx)):
+                    k = axes[idx][1]
+                    v = fidx[idx]
+                    if axes[idx][-1] > 1:
+                        curframe[k] = v
                 frameList.append(curframe)
             if len(frameList) > 1:
                 self._frames = frameList
@@ -210,13 +220,9 @@ def _tileImage(self, rgbColor, x, y, z, frame, band=None, bandnum=0):
         fontsize = 0.15
         text = 'x=%d\ny=%d\nz=%d' % (x, y, z)
         if hasattr(self, '_frames'):
-            if self._framesParts == 1:
-                text += '\nf=%d' % frame
-            else:
-                for k1, k2 in [('C', 'IndexC'), ('Z', 'IndexZ'),
-                               ('T', 'IndexT'), ('XY', 'IndexXY')]:
-                    if k2 in self._frames[frame]:
-                        text += '\n%s=%d' % (k1, self._frames[frame][k2])
+            for k1, k2, _ in self._axes:
+                if k2 in self._frames[frame]:
+                    text += '\n%s=%d' % (k1.upper(), self._frames[frame][k2])
         text += bandtext
         fontsize = min(fontsize, 0.8 / len(text.split('\n')))
         try:

test/test_files/multi_test_source_axes.yml

@@ -0,0 +1,18 @@
+---
+axes:
+  - c
+  - i
+  - j
+sources:
+  - sourceName: test
+    path: __none__
+    params:
+      minLevel: 0
+      maxLevel: 6
+      tileWidth: 256
+      tileHeight: 256
+      sizeX: 10000
+      sizeY: 7500
+      fractal: True
+      frames: "c=4,i=5,j=3"
+      monochrome: False

test/test_source_multi.py

@@ -205,3 +205,17 @@ def testMultiComposite():
     assert len(tileMetadata['frames']) == 116
 
     utilities.checkTilesZXY(source, tileMetadata)
+
+
+def testTilesWithMoreAxes():
+    testDir = os.path.dirname(os.path.realpath(__file__))
+    imagePath = os.path.join(testDir, 'test_files', 'multi_test_source_axes.yml')
+    source = large_image_source_multi.open(imagePath)
+    tileMetadata = source.getMetadata()
+    assert tileMetadata['tileWidth'] == 256
+    assert tileMetadata['tileHeight'] == 256
+    assert tileMetadata['sizeX'] == 10000
+    assert tileMetadata['sizeY'] == 7500
+    assert tileMetadata['levels'] == 7
+    assert len(tileMetadata['frames']) == 60
+    utilities.checkTilesZXY(source, tileMetadata)