From 5ce1235a7a7e33ad7bf9910cc92bdb25119833ae Mon Sep 17 00:00:00 2001 From: Ezequiel Valencia Date: Thu, 31 Oct 2024 12:37:54 -0400 Subject: [PATCH] SimResultsLoader as Central Image Object Instead of passing around ImagePlus internally, pass the SimResultsLoader which allows for extra commands to be executed (ex. meta data retrieval) in addition image data. --- .../java/org/vcell/N5/UI/N5ExportTable.java | 6 +- .../vcell/N5/retrieving/LoadingManager.java | 6 +- .../N5/retrieving/SimLoadingListener.java | 2 +- .../vcell/N5/retrieving/SimResultsLoader.java | 125 +++++++++--------- .../N5/retrieving/N5ImageHandlerTest.java | 25 +--- 5 files changed, 78 insertions(+), 86 deletions(-) diff --git a/view-simulation-results/src/main/java/org/vcell/N5/UI/N5ExportTable.java b/view-simulation-results/src/main/java/org/vcell/N5/UI/N5ExportTable.java index 27ad19a..5429c62 100644 --- a/view-simulation-results/src/main/java/org/vcell/N5/UI/N5ExportTable.java +++ b/view-simulation-results/src/main/java/org/vcell/N5/UI/N5ExportTable.java @@ -223,11 +223,11 @@ public void simIsLoading(int itemRow, String exportID) { } @Override - public void simFinishedLoading(int itemRow, String exportID, ImagePlus imagePlus) { - loadingRowsJobID.remove(itemRow); + public void simFinishedLoading(SimResultsLoader loadedResults) { + loadingRowsJobID.remove(loadedResults.rowNumber); exportListTable.repaint(); controlPanel.allowCancel(false); - imagePlus.show(); + loadedResults.getImagePlus().show(); } static class N5ExportTableModel extends AbstractTableModel { diff --git a/view-simulation-results/src/main/java/org/vcell/N5/retrieving/LoadingManager.java b/view-simulation-results/src/main/java/org/vcell/N5/retrieving/LoadingManager.java index bd17b08..7c05c13 100644 --- a/view-simulation-results/src/main/java/org/vcell/N5/retrieving/LoadingManager.java +++ b/view-simulation-results/src/main/java/org/vcell/N5/retrieving/LoadingManager.java @@ -3,7 +3,6 @@ import com.amazonaws.AbortedException; import com.amazonaws.http.timers.client.SdkInterruptedException; import ij.ImagePlus; -import ij.plugin.Duplicator; import org.scijava.log.Logger; import org.vcell.N5.N5ImageHandler; import org.vcell.N5.UI.ControlButtonsPanel; @@ -57,7 +56,8 @@ public void openN5FileDataset(ArrayList filesToOpen, boolean o imagePlus = simResultsLoader.openInMemory(rangeSelector); } } else{ - imagePlus = simResultsLoader.getImgPlusFromN5File(); + simResultsLoader.loadImageFromN5File(); + imagePlus = simResultsLoader.getImagePlus(); } } catch (RuntimeException e) { @@ -149,7 +149,7 @@ public void notifySimIsLoading(SimResultsLoader simResultsLoader) { @Override public void notifySimIsDoneLoading(SimResultsLoader simResultsLoader, ImagePlus imagePlus) { for (SimLoadingListener simLoadingListener: eventListenerList.getListeners(SimLoadingListener.class)){ - simLoadingListener.simFinishedLoading(simResultsLoader.rowNumber, simResultsLoader.exportID, imagePlus); + simLoadingListener.simFinishedLoading(simResultsLoader); } } } diff --git a/view-simulation-results/src/main/java/org/vcell/N5/retrieving/SimLoadingListener.java b/view-simulation-results/src/main/java/org/vcell/N5/retrieving/SimLoadingListener.java index a53f46d..e517f11 100644 --- a/view-simulation-results/src/main/java/org/vcell/N5/retrieving/SimLoadingListener.java +++ b/view-simulation-results/src/main/java/org/vcell/N5/retrieving/SimLoadingListener.java @@ -8,6 +8,6 @@ public interface SimLoadingListener extends EventListener { public void simIsLoading(int itemRow, String exportID); - public void simFinishedLoading(int itemRow, String exportID, ImagePlus imagePlus); + public void simFinishedLoading(SimResultsLoader loadedResult); } diff --git a/view-simulation-results/src/main/java/org/vcell/N5/retrieving/SimResultsLoader.java b/view-simulation-results/src/main/java/org/vcell/N5/retrieving/SimResultsLoader.java index a731e69..fa72d3c 100644 --- a/view-simulation-results/src/main/java/org/vcell/N5/retrieving/SimResultsLoader.java +++ b/view-simulation-results/src/main/java/org/vcell/N5/retrieving/SimResultsLoader.java @@ -8,13 +8,12 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.google.gson.GsonBuilder; +import com.google.gson.internal.LinkedTreeMap; import ij.ImagePlus; -import ij.VirtualStack; import ij.plugin.ContrastEnhancer; import ij.plugin.Duplicator; import net.imglib2.cache.img.CachedCellImg; import net.imglib2.img.display.imagej.ImageJFunctions; -import net.imglib2.img.display.imagej.ImageJVirtualStackFloat; import net.imglib2.type.numeric.real.DoubleType; import org.apache.http.conn.socket.ConnectionSocketFactory; import org.apache.http.conn.ssl.TrustSelfSignedStrategy; @@ -24,7 +23,6 @@ import org.janelia.saalfeldlab.n5.N5KeyValueReader; import org.janelia.saalfeldlab.n5.N5Reader; import org.janelia.saalfeldlab.n5.imglib2.N5Utils; -import org.janelia.saalfeldlab.n5.s3.N5AmazonS3Reader; import org.scijava.log.Logger; import org.vcell.N5.N5ImageHandler; import org.vcell.N5.UI.RangeSelector; @@ -43,22 +41,21 @@ import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; +import java.util.TreeMap; public class SimResultsLoader { private File selectedLocalFile; - private AmazonS3 s3Client; - private String bucketName; - private String s3ObjectKey; private URI uri; private String dataSetChosen; - public String userSetFileName = null; private static final String defaultS3Region = "site2-low"; - private N5Reader n5AmazonS3Reader; - + private N5Reader n5AmazonS3Reader = null; private static final Logger logger = N5ImageHandler.getLogger(SimResultsLoader.class); public static AmazonS3ClientBuilder s3ClientBuilder; + private ImagePlus imagePlus = null; + + + public String userSetFileName = null; public int rowNumber; public String exportID; @@ -83,42 +80,52 @@ public SimResultsLoader(String stringURI, String userSetFileName){ } } + ///////////////////////////////// + // Loading Simulation Results // + /////////////////////////////// + public void createS3ClientAndReader(){ logger.debug("Creating S3 Client with url: " + uri); if (uri.getHost().equals("minikube.remote") || uri.getHost().equals("minikube.island")){ - SSLContext sslContext = null; - try { - sslContext = SSLContexts.custom().loadTrustMaterial(new TrustSelfSignedStrategy()).build(); - } catch (NoSuchAlgorithmException | KeyManagementException | KeyStoreException ex) { - throw new RuntimeException("Custom ssl context not functional.",ex); - } - HostnameVerifier hostnameVerifier = new HostnameVerifier() { - @Override - public boolean verify(String hostname, SSLSession session) { - return true; - } - }; - ConnectionSocketFactory connectionSocketFactory = new SdkTLSSocketFactory(sslContext, hostnameVerifier); - ClientConfiguration clientConfiguration = new ClientConfiguration(); - clientConfiguration.getApacheHttpClientConfig().setSslSocketFactory(connectionSocketFactory); - s3ClientBuilder.setClientConfiguration(clientConfiguration); + allowInsecureConnections(); } + ////////////////////////////////////////////// + // assume it is one of our URLs // + // http://vcell:8000/bucket/object/object2 // + ////////////////////////////////////////////// - // assume it is one of our URLs - // http://vcell:8000/bucket/object/object2 String[] pathSubStrings = uri.getPath().split("/", 3); - this.s3ObjectKey = pathSubStrings[2]; - this.bucketName = pathSubStrings[1]; + String s3ObjectKey = pathSubStrings[2]; + String bucketName = pathSubStrings[1]; s3ClientBuilder.withPathStyleAccessEnabled(true); logger.debug("Creating S3 Client with Path Based Buckets"); s3ClientBuilder.withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials())); s3ClientBuilder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(uri.getScheme() + "://" + uri.getAuthority(), defaultS3Region)); - this.s3Client = s3ClientBuilder.build(); + AmazonS3 s3Client = s3ClientBuilder.build(); S3KeyValueAccess amazonS3KeyValueAccess = new S3KeyValueAccess(s3Client, bucketName, false); n5AmazonS3Reader = new N5KeyValueReader(amazonS3KeyValueAccess, s3ObjectKey, new GsonBuilder(), false); } + private void allowInsecureConnections(){ + SSLContext sslContext = null; + try { + sslContext = SSLContexts.custom().loadTrustMaterial(new TrustSelfSignedStrategy()).build(); + } catch (NoSuchAlgorithmException | KeyManagementException | KeyStoreException ex) { + throw new RuntimeException("Custom ssl context not functional.",ex); + } + HostnameVerifier hostnameVerifier = new HostnameVerifier() { + @Override + public boolean verify(String hostname, SSLSession session) { + return true; + } + }; + ConnectionSocketFactory connectionSocketFactory = new SdkTLSSocketFactory(sslContext, hostnameVerifier); + ClientConfiguration clientConfiguration = new ClientConfiguration(); + clientConfiguration.getApacheHttpClientConfig().setSslSocketFactory(connectionSocketFactory); + s3ClientBuilder.setClientConfiguration(clientConfiguration); + } + /** By default N5 library assumes that S3 URL's are formated in the standard amazon format. That being https://BUCKETNAME.s3.REGION.amazonaws.com {@link org.janelia.saalfeldlab.n5.s3.AmazonS3Utils}. Because our objects @@ -126,35 +133,42 @@ public boolean verify(String hostname, SSLSession session) { it's the fallback style chosen by the N5 libraries if standard format is unavailable. */ - public ImagePlus getImgPlusFromN5File() { + public ImagePlus getImagePlus(){ + if (imagePlus != null){ + return imagePlus; + } + if (n5AmazonS3Reader == null){ + createS3ClientAndReader(); + } + loadImageFromN5File(); + return imagePlus; + } -// N5AmazonS3Reader n5AmazonS3Reader = new N5AmazonS3Reader(s3Client, bucketName, "/" + s3ObjectKey); + public void loadImageFromN5File() { long start = System.currentTimeMillis(); logger.debug("Reading N5 File " + userSetFileName + " Into Virtual Image"); if (userSetFileName == null || userSetFileName.isEmpty()){ userSetFileName = n5AmazonS3Reader.getAttribute(dataSetChosen, "name", String.class); } - SimCacheLoader simCacheLoader = SimCacheLoader.factoryDefault(n5AmazonS3Reader, dataSetChosen); CachedCellImg cachedCellImg = simCacheLoader.createCachedCellImage(); - ImagePlus imagePlus = ImageJFunctions.wrap(cachedCellImg, userSetFileName); + imagePlus = ImageJFunctions.wrap(cachedCellImg, userSetFileName); simCacheLoader.setImagesResponsibleFor(imagePlus); long end = System.currentTimeMillis(); logger.debug("Read N5 File " + userSetFileName + " Into ImageJ taking: " + ((end - start) / 1000.0) + "s"); setUnits(n5AmazonS3Reader, imagePlus); - imagePlus.setProperty("channelInfo", n5AmazonS3Reader.getAttribute(dataSetChosen, "channelInfo", HashMap.class)); + imagePlus.setProperty("channelInfo", getChannelInfo()); imagePlus.setProperty("maskInfo", n5AmazonS3Reader.getAttribute(dataSetChosen, "maskMapping", HashMap.class)); imagePlus.setZ(Math.floorDiv(imagePlus.getNSlices(), 2)); imagePlus.setT(Math.floorDiv(imagePlus.getNFrames(), 2)); new ContrastEnhancer().stretchHistogram(imagePlus, 1); - return imagePlus; } public ImagePlus openInMemory(RangeSelector rangeSelector){ - ImagePlus imagePlus = this.getImgPlusFromN5File(); + loadImageFromN5File(); long start = System.currentTimeMillis(); logger.debug("Loading Virtual N5 File " + userSetFileName + " Into Memory"); imagePlus = new Duplicator().run(imagePlus, rangeSelector.startC, rangeSelector.endC, rangeSelector.startZ, @@ -164,6 +178,11 @@ public ImagePlus openInMemory(RangeSelector rangeSelector){ return imagePlus; } + + //////////////// + // Properties // + //////////////// + private void setUnits(N5Reader n5Reader, ImagePlus imagePlus){ try{ double pixelWidth = n5Reader.getAttribute(dataSetChosen, "pixelWidth", double.class); @@ -186,38 +205,24 @@ public ArrayList getN5Dimensions(){ return n5AmazonS3Reader.getAttribute(dataSetChosen, "dimensions", ArrayList.class); } + public LinkedTreeMap> getChannelInfo(){ + return (LinkedTreeMap>) n5AmazonS3Reader.getAttribute(dataSetChosen, "channelInfo", LinkedTreeMap.class); + } + void setDataSetChosen(String dataSetChosen) { this.dataSetChosen = dataSetChosen; } + ///////////////////////// + // Local File Reading // + //////////////////////// + void setSelectedLocalFile(File selectedLocalFile){ this.selectedLocalFile = selectedLocalFile; } - ArrayList getS3N5DatasetList() throws IOException { - // used as a flag to tell that remote access is occurring, and that there is no local files - try(N5AmazonS3Reader n5AmazonS3Reader = new N5AmazonS3Reader(this.s3Client, this.bucketName)) { - return new ArrayList<>(Arrays.asList(n5AmazonS3Reader.deepListDatasets(this.s3ObjectKey))); - } - } ImagePlus getImgPlusFromLocalN5File() throws IOException { N5Reader n5Reader = new N5FSReader(selectedLocalFile.getPath()); return ImageJFunctions.wrap((CachedCellImg) N5Utils.open(n5Reader, dataSetChosen), userSetFileName); } - - ArrayList getN5DatasetList() throws IOException { - // auto closes reader - logger.debug("Getting List of N5 Datasets"); - try (N5FSReader n5Reader = new N5FSReader(selectedLocalFile.getPath())) { - String[] metaList = n5Reader.deepList("/"); - ArrayList fList = new ArrayList<>(); - for (String s : metaList) { - if (n5Reader.datasetExists(s)) { - fList.add(s); - }; - } - logger.debug("Got List of N5 Datasets"); - return fList; - } - } } diff --git a/view-simulation-results/src/test/java/org/vcell/N5/retrieving/N5ImageHandlerTest.java b/view-simulation-results/src/test/java/org/vcell/N5/retrieving/N5ImageHandlerTest.java index 1403cb3..140fd13 100644 --- a/view-simulation-results/src/test/java/org/vcell/N5/retrieving/N5ImageHandlerTest.java +++ b/view-simulation-results/src/test/java/org/vcell/N5/retrieving/N5ImageHandlerTest.java @@ -59,13 +59,6 @@ public void run(){ N5ImageHandler.initializeLogService(); } - @Test - public void testN5DatasetList() throws IOException { - SimResultsLoader simResultsLoader = new SimResultsLoader(); - simResultsLoader.setSelectedLocalFile(this.getTestResourceFiles(n5FileName)); - this.dataSetListTest(simResultsLoader.getN5DatasetList()); - } - @Test public void testGettingImgPlus() throws IOException { SimResultsLoader simResultsLoader = new SimResultsLoader(); @@ -84,7 +77,8 @@ public void testS3AlphaInstance() throws IOException{ for(N5DataSetFile n5DataSetFile : n5DataSetFiles) { SimResultsLoader simResultsLoader = new SimResultsLoader(n5DataSetFile.uri, ""); simResultsLoader.createS3ClientAndReader(); - ImagePlus imagePlus = simResultsLoader.getImgPlusFromN5File(); + simResultsLoader.loadImageFromN5File(); + ImagePlus imagePlus = simResultsLoader.getImagePlus(); //stats that have been preemptively calculated within VCell alphaStatsTest(imagePlus, n5DataSetFile, stats.HISTMAX); @@ -99,7 +93,8 @@ public void testS3AlphaInstanceLoadedIntoMemory() throws IOException { for(N5DataSetFile n5DataSetFile : n5DataSetFiles) { SimResultsLoader simResultsLoader = new SimResultsLoader(n5DataSetFile.uri, ""); simResultsLoader.createS3ClientAndReader(); - ImagePlus imagePlus = simResultsLoader.getImgPlusFromN5File(); + simResultsLoader.loadImageFromN5File(); + ImagePlus imagePlus = simResultsLoader.getImagePlus(); ImagePlus inMemory = new Duplicator().run(imagePlus); for (Object property : imagePlus.getProperties().keySet()){ inMemory.setProperty((String) property, imagePlus.getProperty((String) property)); @@ -120,7 +115,8 @@ public void testUnits() throws IOException { for (N5DataSetFile n5DataSetFile: n5DataSetFiles){ SimResultsLoader simResultsLoader = new SimResultsLoader(n5DataSetFile.uri, ""); simResultsLoader.createS3ClientAndReader(); - ImagePlus imagePlus = simResultsLoader.getImgPlusFromN5File(); + simResultsLoader.loadImageFromN5File(); + ImagePlus imagePlus = simResultsLoader.getImagePlus(); double areaOfPixel = imagePlus.getCalibration().getX(1) * imagePlus.getCalibration().getY(1); double totalArea = areaOfPixel * imagePlus.getWidth() * imagePlus.getHeight(); @@ -287,15 +283,6 @@ private ImagePlus setImageMask(ImagePlus imagePlus){ return imagePlus; } - - private void remoteN5ImgPlusTests(SimResultsLoader simResultsLoader) throws IOException { - ImagePlus imagePlus = simResultsLoader.getImgPlusFromN5File(); - dataSetListTest(simResultsLoader.getS3N5DatasetList()); - fiveDStackTests(imagePlus); - imagePlus = new Duplicator().run(imagePlus); //Tests taking the N5 file from streaming to in memory - fiveDStackTests(imagePlus); - } - private void fiveDStackTests(ImagePlus variableImgPlus){ ImagePlus controlImgPlus = Opener.openUsingBioFormats(this.getTestResourceFiles("mitosis.tif").getAbsolutePath()); int[] variableDimensions = variableImgPlus.getDimensions();