Skip to content

Commit

Permalink
SimResultsLoader as Central Image Object
Browse files Browse the repository at this point in the history 
Instead of passing around ImagePlus internally, pass the SimResultsLoader which allows for extra commands to be executed (ex. meta data retrieval) in addition image data.
  • Loading branch information
@AvocadoMoon
AvocadoMoon committed Oct 31, 2024
1 parent 1ec5e2f commit 5ce1235
Show file tree
Hide file tree
Showing 5 changed files with 78 additions and 86 deletions.
6 changes: 3 additions & 3 deletions view-simulation-results/src/main/java/org/vcell/N5/UI/N5ExportTable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -223,11 +223,11 @@ public void simIsLoading(int itemRow, String exportID) {
}

@Override
public void simFinishedLoading(int itemRow, String exportID, ImagePlus imagePlus) {
loadingRowsJobID.remove(itemRow);
public void simFinishedLoading(SimResultsLoader loadedResults) {
loadingRowsJobID.remove(loadedResults.rowNumber);
exportListTable.repaint();
controlPanel.allowCancel(false);
imagePlus.show();
loadedResults.getImagePlus().show();
}

static class N5ExportTableModel extends AbstractTableModel {
Expand Down
6 changes: 3 additions & 3 deletions view-simulation-results/src/main/java/org/vcell/N5/retrieving/LoadingManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
import com.amazonaws.AbortedException;
import com.amazonaws.http.timers.client.SdkInterruptedException;
import ij.ImagePlus;
import ij.plugin.Duplicator;
import org.scijava.log.Logger;
import org.vcell.N5.N5ImageHandler;
import org.vcell.N5.UI.ControlButtonsPanel;
Expand Down Expand Up @@ -57,7 +56,8 @@ public void openN5FileDataset(ArrayList<SimResultsLoader> filesToOpen, boolean o
imagePlus = simResultsLoader.openInMemory(rangeSelector);
}
} else{
imagePlus = simResultsLoader.getImgPlusFromN5File();
simResultsLoader.loadImageFromN5File();
imagePlus = simResultsLoader.getImagePlus();
}
}
catch (RuntimeException e) {
Expand Down Expand Up @@ -149,7 +149,7 @@ public void notifySimIsLoading(SimResultsLoader simResultsLoader) {
@Override
public void notifySimIsDoneLoading(SimResultsLoader simResultsLoader, ImagePlus imagePlus) {
for (SimLoadingListener simLoadingListener: eventListenerList.getListeners(SimLoadingListener.class)){
simLoadingListener.simFinishedLoading(simResultsLoader.rowNumber, simResultsLoader.exportID, imagePlus);
simLoadingListener.simFinishedLoading(simResultsLoader);
}
}
}
2 changes: 1 addition & 1 deletion view-simulation-results/src/main/java/org/vcell/N5/retrieving/SimLoadingListener.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,6 @@ public interface SimLoadingListener extends EventListener {

public void simIsLoading(int itemRow, String exportID);

public void simFinishedLoading(int itemRow, String exportID, ImagePlus imagePlus);
public void simFinishedLoading(SimResultsLoader loadedResult);

}
125 changes: 65 additions & 60 deletions view-simulation-results/src/main/java/org/vcell/N5/retrieving/SimResultsLoader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,12 @@
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.google.gson.GsonBuilder;
import com.google.gson.internal.LinkedTreeMap;
import ij.ImagePlus;
import ij.VirtualStack;
import ij.plugin.ContrastEnhancer;
import ij.plugin.Duplicator;
import net.imglib2.cache.img.CachedCellImg;
import net.imglib2.img.display.imagej.ImageJFunctions;
import net.imglib2.img.display.imagej.ImageJVirtualStackFloat;
import net.imglib2.type.numeric.real.DoubleType;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
Expand All @@ -24,7 +23,6 @@
import org.janelia.saalfeldlab.n5.N5KeyValueReader;
import org.janelia.saalfeldlab.n5.N5Reader;
import org.janelia.saalfeldlab.n5.imglib2.N5Utils;
import org.janelia.saalfeldlab.n5.s3.N5AmazonS3Reader;
import org.scijava.log.Logger;
import org.vcell.N5.N5ImageHandler;
import org.vcell.N5.UI.RangeSelector;
Expand All @@ -43,22 +41,21 @@
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.TreeMap;

public class SimResultsLoader {
private File selectedLocalFile;
private AmazonS3 s3Client;
private String bucketName;
private String s3ObjectKey;
private URI uri;
private String dataSetChosen;
public String userSetFileName = null;
private static final String defaultS3Region = "site2-low";
private N5Reader n5AmazonS3Reader;

private N5Reader n5AmazonS3Reader = null;
private static final Logger logger = N5ImageHandler.getLogger(SimResultsLoader.class);
public static AmazonS3ClientBuilder s3ClientBuilder;
private ImagePlus imagePlus = null;


public String userSetFileName = null;
public int rowNumber;
public String exportID;

Expand All @@ -83,78 +80,95 @@ public SimResultsLoader(String stringURI, String userSetFileName){
}
}

/////////////////////////////////
// Loading Simulation Results //
///////////////////////////////

public void createS3ClientAndReader(){
logger.debug("Creating S3 Client with url: " + uri);
if (uri.getHost().equals("minikube.remote") || uri.getHost().equals("minikube.island")){
SSLContext sslContext = null;
try {
sslContext = SSLContexts.custom().loadTrustMaterial(new TrustSelfSignedStrategy()).build();
} catch (NoSuchAlgorithmException | KeyManagementException | KeyStoreException ex) {
throw new RuntimeException("Custom ssl context not functional.",ex);
}
HostnameVerifier hostnameVerifier = new HostnameVerifier() {
@Override
public boolean verify(String hostname, SSLSession session) {
return true;
}
};
ConnectionSocketFactory connectionSocketFactory = new SdkTLSSocketFactory(sslContext, hostnameVerifier);
ClientConfiguration clientConfiguration = new ClientConfiguration();
clientConfiguration.getApacheHttpClientConfig().setSslSocketFactory(connectionSocketFactory);
s3ClientBuilder.setClientConfiguration(clientConfiguration);
allowInsecureConnections();
}
//////////////////////////////////////////////
// assume it is one of our URLs //
// http://vcell:8000/bucket/object/object2 //
//////////////////////////////////////////////

// assume it is one of our URLs
// http://vcell:8000/bucket/object/object2
String[] pathSubStrings = uri.getPath().split("/", 3);
this.s3ObjectKey = pathSubStrings[2];
this.bucketName = pathSubStrings[1];
String s3ObjectKey = pathSubStrings[2];
String bucketName = pathSubStrings[1];
s3ClientBuilder.withPathStyleAccessEnabled(true);
logger.debug("Creating S3 Client with Path Based Buckets");

s3ClientBuilder.withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials()));
s3ClientBuilder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(uri.getScheme() + "://" + uri.getAuthority(), defaultS3Region));
this.s3Client = s3ClientBuilder.build();
AmazonS3 s3Client = s3ClientBuilder.build();
S3KeyValueAccess amazonS3KeyValueAccess = new S3KeyValueAccess(s3Client, bucketName, false);
n5AmazonS3Reader = new N5KeyValueReader(amazonS3KeyValueAccess, s3ObjectKey, new GsonBuilder(), false);
}

private void allowInsecureConnections(){
SSLContext sslContext = null;
try {
sslContext = SSLContexts.custom().loadTrustMaterial(new TrustSelfSignedStrategy()).build();
} catch (NoSuchAlgorithmException | KeyManagementException | KeyStoreException ex) {
throw new RuntimeException("Custom ssl context not functional.",ex);
}
HostnameVerifier hostnameVerifier = new HostnameVerifier() {
@Override
public boolean verify(String hostname, SSLSession session) {
return true;
}
};
ConnectionSocketFactory connectionSocketFactory = new SdkTLSSocketFactory(sslContext, hostnameVerifier);
ClientConfiguration clientConfiguration = new ClientConfiguration();
clientConfiguration.getApacheHttpClientConfig().setSslSocketFactory(connectionSocketFactory);
s3ClientBuilder.setClientConfiguration(clientConfiguration);
}

/**
By default N5 library assumes that S3 URL's are formated in the standard amazon format. That being
https://BUCKETNAME.s3.REGION.amazonaws.com {@link org.janelia.saalfeldlab.n5.s3.AmazonS3Utils}. Because our objects
don't originate from amazon this is not a format we can possibly mimic, so we have to use path based buckets because
it's the fallback style chosen by the N5 libraries if standard format is unavailable.
*/

public ImagePlus getImgPlusFromN5File() {
public ImagePlus getImagePlus(){
if (imagePlus != null){
return imagePlus;
}
if (n5AmazonS3Reader == null){
createS3ClientAndReader();
}
loadImageFromN5File();
return imagePlus;
}

// N5AmazonS3Reader n5AmazonS3Reader = new N5AmazonS3Reader(s3Client, bucketName, "/" + s3ObjectKey);
public void loadImageFromN5File() {
long start = System.currentTimeMillis();
logger.debug("Reading N5 File " + userSetFileName + " Into Virtual Image");
if (userSetFileName == null || userSetFileName.isEmpty()){
userSetFileName = n5AmazonS3Reader.getAttribute(dataSetChosen, "name", String.class);
}


SimCacheLoader<DoubleType, ?> simCacheLoader = SimCacheLoader.factoryDefault(n5AmazonS3Reader, dataSetChosen);
CachedCellImg<DoubleType, ?> cachedCellImg = simCacheLoader.createCachedCellImage();
ImagePlus imagePlus = ImageJFunctions.wrap(cachedCellImg, userSetFileName);
imagePlus = ImageJFunctions.wrap(cachedCellImg, userSetFileName);
simCacheLoader.setImagesResponsibleFor(imagePlus);
long end = System.currentTimeMillis();
logger.debug("Read N5 File " + userSetFileName + " Into ImageJ taking: " + ((end - start) / 1000.0) + "s");

setUnits(n5AmazonS3Reader, imagePlus);
imagePlus.setProperty("channelInfo", n5AmazonS3Reader.getAttribute(dataSetChosen, "channelInfo", HashMap.class));
imagePlus.setProperty("channelInfo", getChannelInfo());
imagePlus.setProperty("maskInfo", n5AmazonS3Reader.getAttribute(dataSetChosen, "maskMapping", HashMap.class));
imagePlus.setZ(Math.floorDiv(imagePlus.getNSlices(), 2));
imagePlus.setT(Math.floorDiv(imagePlus.getNFrames(), 2));

new ContrastEnhancer().stretchHistogram(imagePlus, 1);
return imagePlus;
}

public ImagePlus openInMemory(RangeSelector rangeSelector){
ImagePlus imagePlus = this.getImgPlusFromN5File();
loadImageFromN5File();
long start = System.currentTimeMillis();
logger.debug("Loading Virtual N5 File " + userSetFileName + " Into Memory");
imagePlus = new Duplicator().run(imagePlus, rangeSelector.startC, rangeSelector.endC, rangeSelector.startZ,
Expand All @@ -164,6 +178,11 @@ public ImagePlus openInMemory(RangeSelector rangeSelector){
return imagePlus;
}


////////////////
// Properties //
////////////////

private void setUnits(N5Reader n5Reader, ImagePlus imagePlus){
try{
double pixelWidth = n5Reader.getAttribute(dataSetChosen, "pixelWidth", double.class);
Expand All @@ -186,38 +205,24 @@ public ArrayList getN5Dimensions(){
return n5AmazonS3Reader.getAttribute(dataSetChosen, "dimensions", ArrayList.class);
}

public LinkedTreeMap<String, LinkedTreeMap<String, String>> getChannelInfo(){
return (LinkedTreeMap<String, LinkedTreeMap<String, String>>) n5AmazonS3Reader.getAttribute(dataSetChosen, "channelInfo", LinkedTreeMap.class);
}

void setDataSetChosen(String dataSetChosen) {
this.dataSetChosen = dataSetChosen;
}

/////////////////////////
// Local File Reading //
////////////////////////

void setSelectedLocalFile(File selectedLocalFile){
this.selectedLocalFile = selectedLocalFile;
}
ArrayList<String> getS3N5DatasetList() throws IOException {

// used as a flag to tell that remote access is occurring, and that there is no local files
try(N5AmazonS3Reader n5AmazonS3Reader = new N5AmazonS3Reader(this.s3Client, this.bucketName)) {
return new ArrayList<>(Arrays.asList(n5AmazonS3Reader.deepListDatasets(this.s3ObjectKey)));
}
}
ImagePlus getImgPlusFromLocalN5File() throws IOException {
N5Reader n5Reader = new N5FSReader(selectedLocalFile.getPath());
return ImageJFunctions.wrap((CachedCellImg<DoubleType, ?>) N5Utils.open(n5Reader, dataSetChosen), userSetFileName);
}

ArrayList<String> getN5DatasetList() throws IOException {
// auto closes reader
logger.debug("Getting List of N5 Datasets");
try (N5FSReader n5Reader = new N5FSReader(selectedLocalFile.getPath())) {
String[] metaList = n5Reader.deepList("/");
ArrayList<String> fList = new ArrayList<>();
for (String s : metaList) {
if (n5Reader.datasetExists(s)) {
fList.add(s);
};
}
logger.debug("Got List of N5 Datasets");
return fList;
}
}
}
25 changes: 6 additions & 19 deletions view-simulation-results/src/test/java/org/vcell/N5/retrieving/N5ImageHandlerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,13 +59,6 @@ public void run(){
N5ImageHandler.initializeLogService();
}

@Test
public void testN5DatasetList() throws IOException {
SimResultsLoader simResultsLoader = new SimResultsLoader();
simResultsLoader.setSelectedLocalFile(this.getTestResourceFiles(n5FileName));
this.dataSetListTest(simResultsLoader.getN5DatasetList());
}

@Test
public void testGettingImgPlus() throws IOException {
SimResultsLoader simResultsLoader = new SimResultsLoader();
Expand All @@ -84,7 +77,8 @@ public void testS3AlphaInstance() throws IOException{
for(N5DataSetFile n5DataSetFile : n5DataSetFiles) {
SimResultsLoader simResultsLoader = new SimResultsLoader(n5DataSetFile.uri, "");
simResultsLoader.createS3ClientAndReader();
ImagePlus imagePlus = simResultsLoader.getImgPlusFromN5File();
simResultsLoader.loadImageFromN5File();
ImagePlus imagePlus = simResultsLoader.getImagePlus();

//stats that have been preemptively calculated within VCell
alphaStatsTest(imagePlus, n5DataSetFile, stats.HISTMAX);
Expand All @@ -99,7 +93,8 @@ public void testS3AlphaInstanceLoadedIntoMemory() throws IOException {
for(N5DataSetFile n5DataSetFile : n5DataSetFiles) {
SimResultsLoader simResultsLoader = new SimResultsLoader(n5DataSetFile.uri, "");
simResultsLoader.createS3ClientAndReader();
ImagePlus imagePlus = simResultsLoader.getImgPlusFromN5File();
simResultsLoader.loadImageFromN5File();
ImagePlus imagePlus = simResultsLoader.getImagePlus();
ImagePlus inMemory = new Duplicator().run(imagePlus);
for (Object property : imagePlus.getProperties().keySet()){
inMemory.setProperty((String) property, imagePlus.getProperty((String) property));
Expand All @@ -120,7 +115,8 @@ public void testUnits() throws IOException {
for (N5DataSetFile n5DataSetFile: n5DataSetFiles){
SimResultsLoader simResultsLoader = new SimResultsLoader(n5DataSetFile.uri, "");
simResultsLoader.createS3ClientAndReader();
ImagePlus imagePlus = simResultsLoader.getImgPlusFromN5File();
simResultsLoader.loadImageFromN5File();
ImagePlus imagePlus = simResultsLoader.getImagePlus();
double areaOfPixel = imagePlus.getCalibration().getX(1) * imagePlus.getCalibration().getY(1);
double totalArea = areaOfPixel * imagePlus.getWidth() * imagePlus.getHeight();

Expand Down Expand Up @@ -287,15 +283,6 @@ private ImagePlus setImageMask(ImagePlus imagePlus){
return imagePlus;
}


private void remoteN5ImgPlusTests(SimResultsLoader simResultsLoader) throws IOException {
ImagePlus imagePlus = simResultsLoader.getImgPlusFromN5File();
dataSetListTest(simResultsLoader.getS3N5DatasetList());
fiveDStackTests(imagePlus);
imagePlus = new Duplicator().run(imagePlus); //Tests taking the N5 file from streaming to in memory
fiveDStackTests(imagePlus);
}

private void fiveDStackTests(ImagePlus variableImgPlus){
ImagePlus controlImgPlus = Opener.openUsingBioFormats(this.getTestResourceFiles("mitosis.tif").getAbsolutePath());
int[] variableDimensions = variableImgPlus.getDimensions();
Expand Down

0 comments on commit 5ce1235

Please sign in to comment.