From 2086b94517543b8e868bd75622b50af0d944d513 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Tue, 29 Aug 2023 17:31:10 -0500 Subject: [PATCH 1/2] First try at YXC storage with --keep-rgb option --- .../bioformats2raw/Converter.java | 102 ++++++++++++++++-- 1 file changed, 93 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java b/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java index d3fb5e16..81a0f52b 100644 --- a/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java +++ b/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java @@ -210,6 +210,9 @@ public class Converter implements Callable { private IProgressListener progressListener; + /** Whether or not to preserve channel storage for RGB data. */ + private boolean keepRGB = false; + // Option setters /** @@ -799,6 +802,21 @@ public void setDimensionOrder(DimensionOrder order) { dimensionOrder = order; } + /** + * Set whether or not to preserve channel ordering for input RGB data. + * False by default, so channels will be separated. + * + * @param keep true if RGB storage should be preserved + */ + @Option( + names = "--keep-rgb", + description = "Keep RGB data (if present) as RGB", + defaultValue = "false" + ) + public void setKeepRGB(boolean keep) { + keepRGB = keep; + } + // Option getters /** @@ -1036,12 +1054,19 @@ public int getMinImageSize() { } /** - * @return current dimension roder + * @return current dimension order */ public DimensionOrder getDimensionOrder() { return dimensionOrder; } + /** + * @return whether or not to preserve RGB data + */ + public boolean getKeepRGB() { + return keepRGB; + } + // Conversion methods /** @@ -1198,17 +1223,20 @@ public void convert() memoizer.setFlattenedResolutions(false); memoizer.setMetadataFiltered(true); memoizer.setMetadataStore(createMetadata()); - ChannelSeparator separator = new ChannelSeparator(memoizer); - separator.setId(inputPath.toString()); - separator.setResolution(0); + IFormatReader wrappedReader = memoizer; + if (!getKeepRGB()) { + wrappedReader = new ChannelSeparator(wrappedReader); + } + wrappedReader.setId(inputPath.toString()); + wrappedReader.setResolution(0); if (reader instanceof MiraxReader) { ((MiraxReader) reader).setTileCache(tileCache); } if (omeroMetadata) { - readers.add(new MinMaxCalculator(separator)); + readers.add(new MinMaxCalculator(wrappedReader)); } else { - readers.add(separator); + readers.add(wrappedReader); } savedMemoFile = savedMemoFile || memoizer.isSavedToMemo(); } @@ -1655,6 +1683,7 @@ private byte[] getTileDownsampled( int bytesPerPixel = FormatTools.getBytesPerPixel(pixelType); int[] shape = new int[] {1, 1, 1, height, width}; + // TODO: fix shape and downsampling for RGB byte[] tileAsBytes = readAsBytes(zarr, shape, offset); if (downsampling == Downsampling.SIMPLE) { @@ -1729,6 +1758,19 @@ private int[] getDimensions( dimensions[o.indexOf("Z")] = sizeZ; dimensions[o.indexOf("C")] = sizeC; dimensions[o.indexOf("T")] = sizeT; + + if (getKeepRGB() && reader.getRGBChannelCount() > 1) { + if (reader.isInterleaved()) { + dimensions[1] = scaledDepth; + dimensions[2] = scaledHeight; + dimensions[3] = scaledWidth; + dimensions[4] = reader.getRGBChannelCount(); + } + else { + dimensions[1] = scaledDepth; + dimensions[2] = reader.getRGBChannelCount(); + } + } return dimensions; } @@ -1753,6 +1795,14 @@ private int[] getOffset( offset[o.indexOf("Z")] = zct[0]; offset[o.indexOf("C")] = zct[1]; offset[o.indexOf("T")] = zct[2]; + + if (getKeepRGB() && reader.getRGBChannelCount() > 1) { + if (reader.isInterleaved()) { + offset[2] = y; + offset[3] = x; + offset[4] = 0; + } + } return offset; } @@ -1845,6 +1895,8 @@ public void saveResolutions(int series) int sizeC; int imageCount; String readerDimensionOrder; + int rgbChannels; + boolean interleaved; try { // calculate a reasonable pyramid depth if not specified as an argument sizeX = workingReader.getSizeX(); @@ -1874,10 +1926,18 @@ public void saveResolutions(int series) LOGGER.info("Using {} pyramid resolutions", resolutions); sizeZ = workingReader.getSizeZ(); sizeT = workingReader.getSizeT(); - sizeC = workingReader.getSizeC(); + sizeC = workingReader.getEffectiveSizeC(); readerDimensionOrder = workingReader.getDimensionOrder(); imageCount = workingReader.getImageCount(); pixelType = workingReader.getPixelType(); + + rgbChannels = workingReader.getRGBChannelCount(); + interleaved = workingReader.isInterleaved(); + + if (rgbChannels != workingReader.getSizeC() && getKeepRGB()) { + throw new UnsupportedOperationException( + "Mixed channel types not supported with '--keep-rgb'"); + } } finally { readers.put(workingReader); @@ -1943,14 +2003,27 @@ public void saveResolutions(int series) activeChunkDepth = scaledDepth; } + int[] chunkSize = new int[] { + 1, 1, activeChunkDepth, activeTileHeight, activeTileWidth}; + if (getKeepRGB() && rgbChannels > 1) { + chunkSize[1] = activeChunkDepth; + if (interleaved) { + chunkSize[2] = activeTileHeight; + chunkSize[3] = activeTileWidth; + chunkSize[4] = rgbChannels; + } + else { + chunkSize[2] = rgbChannels; + } + } + DataType dataType = getZarrType(pixelType); String resolutionString = String.format( scaleFormatString, getScaleFormatStringArgs(series, resolution)); ArrayParams arrayParams = new ArrayParams() .shape(getDimensions( workingReader, scaledWidth, scaledHeight, scaledDepth)) - .chunks(new int[] {1, 1, activeChunkDepth, activeTileHeight, - activeTileWidth}) + .chunks(chunkSize) .dataType(dataType) .dimensionSeparator(getDimensionSeparator()) .compressor(CompressorFactory.create( @@ -1989,6 +2062,14 @@ public void saveResolutions(int series) executor.execute(() -> { try { int[] shape = {1, 1, 1, height, width}; + if (getKeepRGB() && rgbChannels > 1) { + if (interleaved) { + shape = new int[] {1, 1, height, width, rgbChannels}; + } + else { + shape = new int[] {1, 1, rgbChannels, height, width}; + } + } int[] offset; IFormatReader reader = readers.take(); try { @@ -2256,6 +2337,9 @@ private void setSeriesLevelMetadata(int series, int resolutions) else { axisOrder = v.getDimensionOrder(); } + if (getKeepRGB() && v.isInterleaved()) { + axisOrder = "C" + axisOrder.replaceAll("[Cc]", ""); + } } finally { readers.put(v); From 8a201f8a81d28bfdb64d0637817f4eb129de9820 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Tue, 29 Aug 2023 17:51:12 -0500 Subject: [PATCH 2/2] Add RGB downsampling Only simple downsapmling implemented, OpenCV types will throw an exception. --- .../bioformats2raw/Converter.java | 43 ++++++++++++++++--- 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java b/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java index 81a0f52b..301b3076 100644 --- a/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java +++ b/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java @@ -1656,10 +1656,30 @@ private byte[] getTileDownsampled( String.format(scaleFormatString, getScaleFormatStringArgs(series, resolution - 1)); final ZarrArray zarr = ZarrArray.open(getRootPath().resolve(pathName)); + int[] dimensions = zarr.getShape(); int[] blockSizes = zarr.getChunks(); - int activeTileWidth = blockSizes[blockSizes.length - 1]; - int activeTileHeight = blockSizes[blockSizes.length - 2]; + + boolean interleaved = false; + IFormatReader r = readers.take(); + try { + interleaved = r.isInterleaved(); + } + finally { + readers.put(r); + } + + int xIndex = blockSizes.length - 1; + int yIndex = blockSizes.length - 2; + int channels = 1; + if (getKeepRGB() && interleaved) { + xIndex--; + yIndex--; + channels = blockSizes[blockSizes.length - 1]; + } + + int activeTileWidth = blockSizes[xIndex]; + int activeTileHeight = blockSizes[yIndex]; // Upscale our base X and Y offsets, and sizes to the previous resolution // based on the pyramid scaling factor @@ -1667,10 +1687,10 @@ private byte[] getTileDownsampled( yy *= PYRAMID_SCALE; width = (int) Math.min( activeTileWidth * PYRAMID_SCALE, - dimensions[dimensions.length - 1] - xx); + dimensions[xIndex] - xx); height = (int) Math.min( activeTileHeight * PYRAMID_SCALE, - dimensions[dimensions.length - 2] - yy); + dimensions[yIndex] - yy); IFormatReader reader = readers.take(); int[] offset; @@ -1683,14 +1703,18 @@ private byte[] getTileDownsampled( int bytesPerPixel = FormatTools.getBytesPerPixel(pixelType); int[] shape = new int[] {1, 1, 1, height, width}; - // TODO: fix shape and downsampling for RGB + if (getKeepRGB() && interleaved) { + shape[2] = height; + shape[3] = width; + shape[4] = channels; + } byte[] tileAsBytes = readAsBytes(zarr, shape, offset); if (downsampling == Downsampling.SIMPLE) { return scaler.downsample(tileAsBytes, width, height, PYRAMID_SCALE, bytesPerPixel, false, FormatTools.isFloatingPoint(pixelType), - 1, false); + channels, interleaved); } return OpenCVTools.downsample( @@ -1943,14 +1967,19 @@ public void saveResolutions(int series) readers.put(workingReader); } + boolean opencv = getDownsampling() != Downsampling.SIMPLE; if ((pixelType == FormatTools.INT8 || pixelType == FormatTools.INT32) && - getDownsampling() != Downsampling.SIMPLE && resolutions > 0) + opencv && resolutions > 0) { String type = FormatTools.getPixelTypeString(pixelType); throw new UnsupportedOperationException( "OpenCV does not support downsampling " + type + " data. " + "See https://github.com/opencv/opencv/issues/7862"); } + else if (interleaved && rgbChannels > 1 && opencv) { + throw new UnsupportedOperationException( + "Downsampling RGB data with OpenCV not supported"); + } LOGGER.info( "Preparing to write pyramid sizeX {} (tileWidth: {}) " +