From 1570f0708d10ed792026ea0efbc1f327cd8a14b3 Mon Sep 17 00:00:00 2001 From: felix cc Date: Mon, 6 Mar 2017 14:25:34 +0100 Subject: [PATCH] initial commit --- .gitignore | 4 + README.md | 10 + pom.xml | 131 ++++++++ src/main/java/Ndpi2OmeTif.java | 287 +++++++++++++++++ src/main/java/NdpiTileColorSeparator.java | 372 ++++++++++++++++++++++ 5 files changed, 804 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 pom.xml create mode 100644 src/main/java/Ndpi2OmeTif.java create mode 100644 src/main/java/NdpiTileColorSeparator.java diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..aeb33af --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.idea +*.iml +*/target +update \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..a84f500 --- /dev/null +++ b/README.md @@ -0,0 +1,10 @@ +#NanoZoomer-J +NanoZoomer-J is a collection of [ImageJ][imagej] plugins to deal with the +NDPI image file format. +The "NDPI 2 OME-TIF" converter uses [Bio-Formats][bf] to convert the propriatary +file format to the open ome-tif without loading the entire file into memory, +which is important given that ndpi-files may contain a lot of data. + + +[imagej]: http://imagej.net +[bf]: http://www.openmicroscopy.org/site/products/bio-form… \ No newline at end of file diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..fdba79c --- /dev/null +++ b/pom.xml @@ -0,0 +1,131 @@ + + + + 4.0.0 + + + sc.fiji + pom-fiji + 21.0.0 + + + + jar + + + 1.8 + Felix Meyenhofer + GPLv3 + + + ch.unifr.imagej + nanozoomerj + 1.1.0-SNAPSHOT + NanoZoomer-J + Collection of plugins allowing to pre-process ndpi-files from the NanoZoomer + + + + + 2015 + + + + + GPL3 + http://www.gnu.org/licenses/gpl-3.0.en.html + Common Development and Distribution License (CDDL-1.0) + + + + + + f.meyenhofer + Felix Meyenhofer + f.meyenhofer@me.com + https://github.com/Meyenhofer + +2 + + lead + developer + debugger + reviewer + support + maintainer + + + + + + scm:git://github.com:Meyenhofer/NanoZoomer-J.git + scm:git@github.com:Meyenhofer/NanoZoomer-J.git + HEAD + https://github.com/Meyenhofer/NanoZoomer-J + + + + GitHub Issues + https://github.com/Meyenhofer/NanoZoomer-J/issues + + + + + ImageJ Forum + http://forum.imagej.net + + + + + + imagej.public + http://maven.imagej.net/content/groups/public + + + + + + net.imagej + imagej + + + io.scif + scifio + + + + ome + formats-bsd + 5.1.7 + + + ome + formats-api + 5.1.7 + + + ome + formats-gpl + 5.1.7 + + + ome + formats-common + 5.1.7 + + + ome + bio-formats_plugins + + 5.1.7 + + + gov.nih.imagej + imagej + + + + + + \ No newline at end of file diff --git a/src/main/java/Ndpi2OmeTif.java b/src/main/java/Ndpi2OmeTif.java new file mode 100644 index 0000000..6a24128 --- /dev/null +++ b/src/main/java/Ndpi2OmeTif.java @@ -0,0 +1,287 @@ +import ij.IJ; +import loci.common.services.DependencyException; +import loci.common.services.ServiceException; +import loci.common.services.ServiceFactory; +import loci.formats.*; +import loci.formats.ome.OMEXMLMetadata; +import loci.formats.services.OMEXMLService; +import net.imagej.ImageJ; +import ome.xml.meta.OMEXMLMetadataRoot; +import ome.xml.model.Image; +import ome.xml.model.enums.DimensionOrder; +import ome.xml.model.enums.EnumerationException; +import ome.xml.model.enums.PixelType; +import ome.xml.model.primitives.PositiveInteger; +import org.apache.commons.io.FilenameUtils; +import org.scijava.ItemVisibility; +import org.scijava.command.Command; +import org.scijava.log.LogService; +import org.scijava.plugin.Parameter; +import org.scijava.plugin.Plugin; +import org.scijava.widget.FileWidget; +import org.scijava.widget.NumberWidget; + +import java.io.File; +import java.io.IOException; +import java.text.DecimalFormat; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + + +/** + * Convert a given series from a NDPI file to OME-TIF + * + * @author Felix Meyenhofer + * creation: 13.10.15 + */ +@Plugin(type = Command.class, menuPath = "Plugins > NanoZoomer > NPDI 2 OME-TIF") +public class Ndpi2OmeTif implements Command { + + // Hardcoded configs + private static final String NDPI_EXTENSION = ".ndpi"; + private static final HashMap channelNameMapper; + static + { + channelNameMapper = new HashMap<>(); + channelNameMapper.put("", null); + channelNameMapper.put("DAPI", 2); + channelNameMapper.put("FITC", 1); + channelNameMapper.put("Cy3", 1); + channelNameMapper.put("TRIC", 0); + channelNameMapper.put("Cy5", 0); + } + + + // Input Dialog + @Parameter(style = FileWidget.DIRECTORY_STYLE, label = "Input directory") + private File inputDir; + + @Parameter(style = FileWidget.DIRECTORY_STYLE, label = "Output directory") + private File outputDir; + + // TODO: dialog callbacks (as soon as Curtis provides it) + @Parameter(label = "Series to convert", style = NumberWidget.SPINNER_STYLE, min = "1", max = "5", stepSize = "1") + private int series = 1; + + @Parameter(choices = {"DAPI", "FITC", "Cy3", "TRIC", "Cy5"}, label = "Channel name") + private String channelName; + + @Parameter(label="Use channel name as file filter") + private boolean matchChannelName = true; + + @Parameter(label="File name regexp") + private String regex = ".*(\\d{6})_.*_(\\d{1,2}).*_ROI(\\d{1,3}).*"; + + @Parameter(choices = {"LZW", "None"}, label = "Output file compression") + private String compression = "LZW"; + + @Parameter(visibility = ItemVisibility.MESSAGE) + private final String note = "Note:"; + + + // Services + @Parameter + private LogService logger; + + + /** + * {@inheritDoc} + */ + public void run() { + IJ.run("Console", "uiservice=[org.scijava.ui.DefaultUIService [priority = 0.0]]"); + + // Prepare input + int colorIndex = channelNameMapper.get(channelName); + int seriesIndex = series - 1; + +// File outputDir = new File(inputDir.getParent(), "ome-tif"); +// if (!outputDir.exists()) { +// if (outputDir.mkdir()) { +// logger.info("Created output directory: " + outputDir.getAbsolutePath()); +// } +// } + + // Process file by file + List fileList = getNdpiFileList(inputDir, channelName); + int nfiles = fileList.size(); + int nfile = 1; + logger.info("Input directory: " + inputDir); + logger.info("Found " + fileList.size() + " files."); + for (File file : fileList) { + IJ.showProgress(nfile++, nfiles); + + // Extract information for the input path + String fileName = reComposeFileName(file); + File outputPath = new File(outputDir, fileName); + + logger.info("converting: " + file.getAbsolutePath()); + logger.info(" to: " + outputPath.getAbsolutePath()); + if (outputPath.exists()) { + logger.info(" already processed"); + continue; + } + + try { + convert(file.getAbsolutePath(), seriesIndex, colorIndex, outputPath.getAbsolutePath()); + } catch (IOException | FormatException | ServiceException | DependencyException | EnumerationException e) { + logger.error(e); + } + } + + IJ.showStatus("Converted " + nfiles + " ndpi-files to ome-tiff"); + logger.info("done."); + } + + /** + * If a regular expression is defined the file name is re-arranged + * according to the matched groups + * @param file file name + * @return re-arranged file name + */ + private String reComposeFileName(File file) { + if (regex.isEmpty()) + return FilenameUtils.removeExtension(file.getName()) + ".ome.tif"; + + Pattern pattern = Pattern.compile(regex); + Matcher match = pattern.matcher(file.getName()); + if (!(match.find() && match.groupCount() == 3)) + throw new RuntimeException("The pattern '" + regex + "' could not extract date, slice and roi from the filename"); + + // Create the output file + DecimalFormat formatter = new DecimalFormat("00"); + + return match.group(1) + + "_" + channelName + + "_slice-" + formatter.format(Double.parseDouble(match.group(2))) + + "_roi-" + match.group(3) + ".ome.tif"; + } + + /** + * Convert a a given series of the input file + * @param inId input file path + * @param outSeries series to write to the output file + * @param outColInd color index to be written to the output (0->red, 1->green, 2->blue) + * @param outId output file + * @throws IOException + * @throws FormatException + * @throws DependencyException + * @throws ServiceException + * @throws EnumerationException + */ + private void convert(String inId, int outSeries, int outColInd, String outId) + throws IOException, FormatException, DependencyException, ServiceException, EnumerationException { + // Record metadata to OME-XML format + ServiceFactory factory = new ServiceFactory(); + OMEXMLService service = factory.getInstance(OMEXMLService.class); + OMEXMLMetadata inMeta = service.createOMEXMLMetadata(); + + // Initialize a file reader wrapped in a channel separator + ChannelSeparator channelSeparator = new ChannelSeparator(); + channelSeparator.setMetadataStore(inMeta); + channelSeparator.setId(inId); + channelSeparator.setSeries(outSeries); + + int numCol = 3;//(channelSeparator.isRGB()) ? 3 : 1; + int inPlanes = channelSeparator.getImageCount(); + int outPlanes = (inPlanes >= 3) ? inPlanes / numCol : inPlanes; + + // This would be the code to start from scratch with the meta data +// ServiceFactory factory = new ServiceFactory(); +// OMEXMLService service = factory.getInstance(OMEXMLService.class); +// IMetadata outMeta = service.createOMEXMLMetadata(); +// MetadataTools.populateMetadata(outMeta, +// 0, +// null, +// false, +// "XYZCT", +// FormatTools.getPixelTypeString(FormatTools.UINT8), +// channelSeparator.getSizeX(), +// channelSeparator.getSizeY(), +// channelSeparator.getImageCount() / numCol, +// 1, 1, 1); + + + // Clone the metadata and remove all the series in the metadata except the one we process + OMEXMLMetadata outMeta = service.createOMEXMLMetadata(inMeta.dumpXML()); + OMEXMLMetadataRoot root = (OMEXMLMetadataRoot) outMeta.getRoot(); + List inSeries = root.copyImageList(); + for (int i = 0; i < inSeries.size(); i++) { + if (i != outSeries) { + root.removeImage(inSeries.get(i)); + } + } + outMeta.setRoot(root); + + // Adjust the metadata attributes affected during this process. + outMeta.setImageName(null, 0); + outMeta.setPixelsSizeC(new PositiveInteger(1), 0); + outMeta.setPixelsSizeZ(new PositiveInteger(outPlanes), 0); + outMeta.setPixelsSizeT(new PositiveInteger(1), 0); + outMeta.setPixelsBinDataBigEndian(Boolean.FALSE, 0, 0); + outMeta.setPixelsDimensionOrder(DimensionOrder.fromString("XYZCT"), 0); + outMeta.setChannelSamplesPerPixel(new PositiveInteger(1), 0, 0); + outMeta.setPixelsType(PixelType.fromString(FormatTools.getPixelTypeString(FormatTools.UINT8)), 0); + + // Setup the writer + ImageWriter writer = new ImageWriter(); + writer.setMetadataRetrieve(outMeta); + if (!compression.equals("None")) + writer.setCompression(compression); + writer.setId(outId); + + // TODO: save tile by tile + // Copy the planes + int outPlaneInd = 0; + byte[] img = new byte[channelSeparator.getSizeX() * channelSeparator.getSizeY()]; + for (int inPlaneInd = outColInd; inPlaneInd < inPlanes; inPlaneInd += numCol) { + logger.info(" writing plane: " + outPlaneInd); + channelSeparator.openBytes(inPlaneInd, img); + writer.saveBytes(outPlaneInd++, img); + } + + // Cleanup + channelSeparator.close(); + writer.close(); + } + + /** + * Get the list of ndpi files for a given input directory + * @param inputDir directory containing the input files + * @param filter string that has to be contained in the file name + * @return list of files in inputDir containing the filter string + */ + private List getNdpiFileList(File inputDir, String filter) { + List list = new ArrayList(); + filter = filter.toLowerCase(); + + File[] content = inputDir.listFiles(); + if (content != null) { + for (File file : content) { + if (!matchChannelName || + file.getName().toLowerCase().contains(NDPI_EXTENSION.toLowerCase()) && + file.getName().toLowerCase().contains(filter)) { + list.add(file); + } + } + } + + return list; + } + + /** + * Test + * @param args input arguments + * @throws Exception anything that can go wrong + */ + public static void main(final String... args) throws Exception { + final ImageJ ij = net.imagej.Main.launch(args); + ij.command().run(Ndpi2OmeTif.class, true); + } +} diff --git a/src/main/java/NdpiTileColorSeparator.java b/src/main/java/NdpiTileColorSeparator.java new file mode 100644 index 0000000..a66969f --- /dev/null +++ b/src/main/java/NdpiTileColorSeparator.java @@ -0,0 +1,372 @@ +import ij.ImagePlus; +import ij.process.ByteProcessor; +import io.scif.SCIFIO; +import loci.common.services.*; +import loci.common.services.DependencyException; +import loci.formats.*; +import loci.formats.FormatException; +import loci.formats.meta.IMetadata; +import loci.formats.services.OMEXMLService; +import net.imagej.ImageJ; +import org.apache.commons.io.FilenameUtils; +import org.scijava.ItemVisibility; +import org.scijava.command.Command; +import org.scijava.log.LogService; +import org.scijava.plugin.Parameter; +import org.scijava.plugin.Plugin; +import org.scijava.widget.FileWidget; + +import java.io.File; +import java.io.IOException; +import java.text.DecimalFormat; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Plugin to separate the channels from the NanoZoomer tif tile export generated with the NDP.Toolbox. + * Tile naming: XXXX_YYYY.tif (single layer z) + * ZZZZ_XXXX_YYYY (multilayer z) + * The tiles index starts from the upper left corner. + * + * @author Felix Meyenhofer + * creation: 21.10.15 + */ +@Plugin(type = Command.class, menuPath = "Plugins > NanoZoomer > Tif-Tiles Color Separation") +public class NdpiTileColorSeparator implements Command { + + /** Debug switch */ + private static final boolean DEBUG = false; + + /** Reference to the file list in the input directory */ + private List tileList; + + + // Hardcoded paramter + private static final String TILE_EXTENSION = ".tif"; + private static final int RED = 1; + private static final int GREEN = 2; + private static final int BLUE = 3; + + + // Dialog + @Parameter(visibility = ItemVisibility.MESSAGE) + private final String note = "Process the tiles produced by Hamamatsu's NDP.Toolbox
" + + "The plugin will consolidate all the tiles in one directory
" + + "and write each channel into a separate file.
" + + "The regular expression extracts three groups: date, slice, roi.
" + + "Input an empty string to use the original name."; + + @Parameter(style = FileWidget.DIRECTORY_STYLE, label = "Input directory:") + private File inputDir = new File("/"); + + @Parameter(style = FileWidget.DIRECTORY_STYLE, label = "Output directory:") + private File outputDir; + + @Parameter(label = "Red") + private boolean processRed = true; + + @Parameter(label = "Green") + private boolean processGreen = true; + + @Parameter(label = "Blue") + private boolean processBlue = true; + + @Parameter(label="File name regexp") + private String regex = ".*(\\d{6})_.*_(\\d{1,2}).*_ROI(\\d{1,3}).*"; + + + // Services + @Parameter + private LogService logger; + + @Parameter + SCIFIO scifio; + + + /** + * {@inheritDoc} + */ + public void run() { + ArrayList directories = getSubDirectories(inputDir); + + if (directories == null) { + logger.error("Could not find sub-directories. This plugins expects one directory for each ndpi-file (and ROI) containing the corresponding tif-tiles."); + return; + } + + for (File directory : directories) { + logger.info("Processing tiles in :" + directory.getAbsolutePath()); + + tileList = getTifTiles(directory, ""); + if (tileList == null || tileList.isEmpty()) { + logger.info(" no tiles found."); + } else { + try { + while (!tileList.isEmpty()) { + List stackList = nextStackFileSet(); + if (processRed) + rgbTiffs2GcStack(stackList, RED); + + if (processGreen) + rgbTiffs2GcStack(stackList, GREEN); + + if (processBlue) + rgbTiffs2GcStack(stackList, BLUE); + } + } catch (IOException | ServiceException | DependencyException | FormatException e) { + logger.error(e); + } + } + } + + logger.info("done."); + } + + /** + * If a regular expression is defined the file name is re-arranged + * according to the matched groups + * @param file file name + * @return re-arranged file name + */ + private File getOutputFile(File file, int channel) { + String fileName; + + // If no info is extracted from the input path, just take the input name + if (regex.isEmpty()) { + fileName = file.getParentFile().getName() + "_" + file.getName(); + } else { + + // Extract information from the + Pattern pattern = Pattern.compile(regex); + Matcher match = pattern.matcher(file.getParentFile().getName()); + if (!(match.find() && match.groupCount() == 3)) + throw new RuntimeException("The pattern '" + regex + "' could not extract date, slice and roi from the filename"); + + // Create the output file + DecimalFormat formatter = new DecimalFormat("00"); + + // Extract tile coordinates + String[] parts = FilenameUtils.removeExtension(file.getName()).split("_"); + String x; + String y; + if (parts.length == 2) { + x = parts[0]; + y = parts[1]; + } else if (parts.length == 3) { + x = parts[1]; + y = parts[2]; + } else { + throw new RuntimeException("Could not extract the x-y-z indices from the file name: " + file.getName()); + } + + fileName = match.group(1) + + "_channel-" + channel + + "_slice-" + formatter.format(Double.parseDouble(match.group(2))) + + "_roi-" + match.group(3) + + "_tile-" + x + "-" + y + + ".tif"; + } + + return new File(outputDir, fileName); + } + + /** + * Get all the subdirectories + * @param parentDirectory to look for sub-directories + * @return list of sub-directories + */ + private ArrayList getSubDirectories(File parentDirectory) { + File[] files = parentDirectory.listFiles(); + if (files == null || files.length == 0) + return null; + + ArrayList subDirectories = new ArrayList(); + for (File file : files) + if (file.isDirectory()) + subDirectories.add(file); + + return subDirectories; + } + + /** + * Get all the tile files in a directory + * @param inputDir directory containing the files + * @param filter string that the files names have to contain + * @return list of tile files + */ + private ArrayList getTifTiles(File inputDir, String filter) { + ArrayList list = new ArrayList(); + + File[] content = inputDir.listFiles(); + if (content != null) { + for (File file : content) { + if (file.getName().contains(TILE_EXTENSION) && file.getName().contains(filter)) { + list.add(file); + } + } + } + + return list; + } + + /** + * Get the set of files belonging to the same + * @return get the next files belonging to one tile (several in case it's a stack) + */ + private List nextStackFileSet() { + List files = new ArrayList(); + + String[] refParts = FilenameUtils.removeExtension(tileList.get(0).getName()).split("_"); + if (refParts.length == 2) { // There is one single z-plane + files.add(tileList.get(0)); + tileList.removeAll(files); + return files; + + } else if (refParts.length == 3) { // Multiple z-planes + for (File file : tileList) { + String[] curParts = FilenameUtils.removeExtension(file.getName()).split("_"); + if (curParts.length != 3) { + logger.info("Skipping file: " + file.getAbsolutePath()); + continue; + } + + // check if x and y index of the tile are the same + if (curParts[2].equals(refParts[2]) && curParts[1].equals(refParts[1])) { + files.add(file); + } + } + + tileList.removeAll(files); + return files; + } + + return null; + } + + + + /** + * Takes one or several files and converts the RGB tif to a gray-scale tif/tif-stack + * + * @param inpFiles list of files (one for a single image, several for stacks) + * @param color the channel/color to be extracted + * @throws loci.common.services.DependencyException + * @throws ServiceException + * @throws IOException + * @throws loci.formats.FormatException + */ + private void rgbTiffs2GcStack(List inpFiles, int color) throws + DependencyException, + ServiceException, + IOException, + FormatException { + + File outStack = getOutputFile(inpFiles.get(0), color); + if (outStack.exists()) { + logger.info(" already processed"); + return; + } + + ChannelSeparator channelSeparator = new ChannelSeparator(); + channelSeparator.setId(inpFiles.get(0).getAbsolutePath()); + + int numCol = (channelSeparator.isRGB()) ? 3 : 1; + int colOff = color - 1; + + ServiceFactory factory = new ServiceFactory(); + OMEXMLService service = factory.getInstance(OMEXMLService.class); + IMetadata outMeta = service.createOMEXMLMetadata(); + MetadataTools.populateMetadata(outMeta, + 0, + null, + false, + "XYZCT", + FormatTools.getPixelTypeString(FormatTools.UINT8), + channelSeparator.getSizeX(), + channelSeparator.getSizeY(), + inpFiles.size(), + 1, 1, 1); + + ImageWriter writer = new ImageWriter(); + writer.setMetadataRetrieve(outMeta); + writer.setId(outStack.getAbsolutePath()); + + int planeInd = 0; + + for (File inpFile : inpFiles) { + channelSeparator.setId(inpFile.getAbsolutePath()); + channelSeparator.setSeries(0); + + for (int i = colOff; i < channelSeparator.getImageCount(); i += numCol) { + byte[] img = channelSeparator.openBytes(i); + + if (DEBUG) { + ByteProcessor bytePro = new ByteProcessor(channelSeparator.getSizeX(), + channelSeparator.getSizeY(), img); + ImagePlus chunk = new ImagePlus("plane " + planeInd, bytePro); + chunk.show(); + } + + writer.saveBytes(planeInd++, img); + } + } + + channelSeparator.close(); + writer.close(); + } + +//// Scifio version (could not figure out how to separate the colors) +// /** +// * Write a separate file for each color/channel. +// * @param inpStack +// * @param color +// * @throws IOException +// * @throws FormatException +// * @throws ImgIOException +// */ +// +// private void separateColors(List inpStack, int color) throws IOException, FormatException, ImgIOException { +// +// for (int p = 0; p < inpStack.size(); p++) { +// File file = inpStack.get(p); +// +// +// File outStack = getOutputFile(file, color); +// if (outStack.exists()) { +// logger.info(" already processed"); +// return; +// } +// +// Reader reader = scifio.initializer().initializeReader(file.getAbsolutePath()); +// +// if (DEBUG) { +// ImagePlus imp = IJ.openImage(file.getAbsolutePath()); +// imp.show(); +// } +// +// Writer writer = scifio.initializer().initializeWriter(file.getAbsolutePath(), outStack.getAbsolutePath()); +// for (int i = 0; i < reader.getImageCount(); i++) { +// for (int j = 0; j < reader.getPlaneCount(i); j++) { +// Plane plane = reader.openPlane(i, j); +// +// writer.savePlane(0, p, plane); +// } +// } +// reader.close(); +// writer.close(); +// } +// } + + /** + * Test + * @param args input arguments + * @throws Exception anything that can go wrong + */ + public static void main(final String... args) throws Exception { + final ImageJ ij = net.imagej.Main.launch(args); + ij.command().run(NdpiTileColorSeparator.class, true); + } + +}