virtualcell · jcschaff · Aug 21, 2023 · Aug 22, 2023 · Aug 23, 2023 · Aug 23, 2023
diff --git a/docker/swarm/docker-compose.yml b/docker/swarm/docker-compose.yml
@@ -349,6 +349,56 @@ services:
       placement:
         constraints:
           - node.labels.zone == INTERNAL
+
+  s3proxy:
+    image: andrewgaul/s3proxy:latest
+    ports:
+      - "${VCELL_S3PROXY_PORT}:80"
+#      - "${VCELL_S3PROXY_PORT}:443"
+    environment:
+      - LOG_LEVEL="info"
+      - S3PROXY_AUTHORIZATION="none" #Set it to None means that anyone can access and use this
+#      - S3PROXY_ENDPOINT="http://0.0.0.0:80"
+#      - S3PROXY_IDENTITY="local-identity"
+#      - S3PROXY_CREDENTIAL="local-credential"
+#      - S3PROXY_VIRTUALHOST=""
+#      - S3PROXY_KEYSTORE_PATH="keystore.jks"
+#      - S3PROXY_KEYSTORE_PASSWORD="password"
+      - S3PROXY_CORS_ALLOW_ALL="false"
+#      - S3PROXY_CORS_ALLOW_ORIGINS=""
+#      - S3PROXY_CORS_ALLOW_METHODS=""
+#      - S3PROXY_CORS_ALLOW_HEADERS=""
+      - S3PROXY_IGNORE_UNKNOWN_HEADERS="false"
+#      - S3PROXY_ENCRYPTED_BLOBSTORE=""
+#      - S3PROXY_ENCRYPTED_BLOBSTORE_PASSWORD=""
+#      - S3PROXY_ENCRYPTED_BLOBSTORE_SALT=""
+      - JCLOUDS_PROVIDER="filesystem"
+#      - JCLOUDS_ENDPOINT=""
+#      - JCLOUDS_REGION=""
+#      - JCLOUDS_REGIONS="us-east-1"
+#      - JCLOUDS_IDENTITY="remote-identity"
+#      - JCLOUDS_CREDENTIAL="remote-credential"
+#      - JCLOUDS_KEYSTONE_VERSION=""
+#      - JCLOUDS_KEYSTONE_SCOPE=""
+#      - JCLOUDS_KEYSTONE_PROJECT_DOMAIN_NAME=""
+      - JCLOUDS_FILESYSTEM_BASEDIR="/data"
+    volumes:
+      - "${VCELL_S3PROXY_DIR}:/data"
+    networks:
+      - vcellnet
+    deploy:
+      mode: replicated
+      replicas: 1
+      resources:
+        limits:
+          memory: 2000M
+        reservations:
+          memory: 1000M
+      placement:
+        constraints:
+          - node.labels.zone == INTERNAL
+
+
   mongodb:
     image: "${VCELL_REPO_NAMESPACE}/vcell-mongo:${VCELL_TAG}"
     ports:

diff --git a/docker/swarm/localconfig_realslurm_oracle.sh b/docker/swarm/localconfig_realslurm_oracle.sh
@@ -105,6 +105,7 @@ VCELL_SLURM_CENTRAL_SINGULARITY_DIR=/share/apps/vcell3/singularityImages
 #
 
 VCELL_API_PORT_EXTERNAL=$((8080 + _site_port_offset))
+VCELL_S3PROXY_PORT=$((8100 + _site_port_offset))
 VCELL_JMS_SIM_PORT_EXTERNAL=$((61616 + _site_port_offset))
 VCELL_JMS_SIM_RESTPORT_EXTERNAL=$((8161 + _site_port_offset))
 VCELL_MONGO_PORT_EXTERNAL=$((27017 + _site_port_offset))
@@ -178,6 +179,8 @@ VCELL_OPT_DOCKER_IMAGE=$VCELL_OPT_DOCKER_IMAGE
 VCELL_OPT_SINGULARITY_FILENAME=$VCELL_OPT_SINGULARITY_FILENAME
 VCELL_OPT_SINGULARITY_IMAGE_EXTERNAL=$VCELL_OPT_SINGULARITY_IMAGE_EXTERNAL
 VCELL_REPO_NAMESPACE=$VCELL_REPO_NAMESPACE
+VCELL_S3PROXY_DIR=/Volumes/vcell/s3proxy
+VCELL_S3PROXY_PORT=$VCELL_S3PROXY_PORT
 VCELL_SECRETS_DIR=${HOME}/vcellkeys_oracle
 VCELL_SIMDATADIR_ARCHIVE_EXTERNAL=/share/apps/vcell12/users
 VCELL_SIMDATADIR_ARCHIVE_INTERNAL=/share/apps/vcell12/users

diff --git a/docker/swarm/localconfig_realslurm_postgres.sh b/docker/swarm/localconfig_realslurm_postgres.sh
@@ -105,6 +105,7 @@ VCELL_SLURM_CENTRAL_SINGULARITY_DIR=/share/apps/vcell3/singularityImages
 #
 
 VCELL_API_PORT_EXTERNAL=$((8080 + _site_port_offset))
+VCELL_S3PROXY_PORT=$((8100 + _site_port_offset))
 VCELL_JMS_SIM_PORT_EXTERNAL=$((61616 + _site_port_offset))
 VCELL_JMS_SIM_RESTPORT_EXTERNAL=$((8161 + _site_port_offset))
 VCELL_MONGO_PORT_EXTERNAL=$((27017 + _site_port_offset))
@@ -178,6 +179,8 @@ VCELL_OPT_DOCKER_IMAGE=$VCELL_OPT_DOCKER_IMAGE
 VCELL_OPT_SINGULARITY_FILENAME=$VCELL_OPT_SINGULARITY_FILENAME
 VCELL_OPT_SINGULARITY_IMAGE_EXTERNAL=$VCELL_OPT_SINGULARITY_IMAGE_EXTERNAL
 VCELL_REPO_NAMESPACE=$VCELL_REPO_NAMESPACE
+VCELL_S3PROXY_DIR=/Volumes/vcell/s3proxy
+VCELL_S3PROXY_PORT=$VCELL_S3PROXY_PORT
 VCELL_SECRETS_DIR=${HOME}/vcellkeys_postgres
 VCELL_SIMDATADIR_ARCHIVE_EXTERNAL=/share/apps/vcell12/users
 VCELL_SIMDATADIR_ARCHIVE_INTERNAL=/share/apps/vcell12/users

diff --git a/docker/swarm/serverconfig-uch.sh b/docker/swarm/serverconfig-uch.sh
@@ -40,20 +40,23 @@ case $VCELL_SITE in
 		_site_port_offset=0
 		VCELL_API_HOST_EXTERNAL=vcellapi.cam.uchc.edu
 		VCELL_API_PORT_EXTERNAL=443
+		VCELL_S3PROXY_PORT=8100
 		VCELL_DEBUG_PORT_BASE=5000
 		_applicationId="1471-8022-1038-5553"
 		;;
 	BETA)
 		_site_port_offset=1
 		VCELL_API_HOST_EXTERNAL=vcellapi.cam.uchc.edu
 		VCELL_API_PORT_EXTERNAL=443
+		VCELL_S3PROXY_PORT=8101
 		VCELL_DEBUG_PORT_BASE=5010
 		_applicationId="1471-8022-1038-5552"
 		;;
 	ALPHA)
 		_site_port_offset=2
 		VCELL_API_HOST_EXTERNAL=vcellapi-beta.cam.uchc.edu
 		VCELL_API_PORT_EXTERNAL=8080
+		VCELL_S3PROXY_PORT=8102
 		VCELL_DEBUG_PORT_BASE=5020
 		_applicationId="1471-8022-1038-5554"
 		;;
@@ -62,34 +65,39 @@ case $VCELL_SITE in
 		# VCELL_API_PORT_EXTERNAL=8081
 		VCELL_API_HOST_EXTERNAL=vcellapi-beta.cam.uchc.edu
 		VCELL_API_PORT_EXTERNAL=443
+		VCELL_S3PROXY_PORT=8103
 		VCELL_DEBUG_PORT_BASE=5030
 		_applicationId="1471-8022-1038-5555"
 		;;
 	TEST2)
 		_site_port_offset=4
 		VCELL_API_HOST_EXTERNAL=vcellapi-beta.cam.uchc.edu
 		VCELL_API_PORT_EXTERNAL=8082
+		VCELL_S3PROXY_PORT=8104
 		VCELL_DEBUG_PORT_BASE=5040
 		_applicationId="1471-8022-1038-5556"
 		;;
 	TEST3)
 		_site_port_offset=5
 		VCELL_API_HOST_EXTERNAL=vcellapi-beta.cam.uchc.edu
 		VCELL_API_PORT_EXTERNAL=8083
+		VCELL_S3PROXY_PORT=8105
 		VCELL_DEBUG_PORT_BASE=5050
 		_applicationId="1471-8022-1038-5557"
 		;;
 	TEST4)
 		_site_port_offset=6
 		VCELL_API_HOST_EXTERNAL=vcellapi-beta.cam.uchc.edu
 		VCELL_API_PORT_EXTERNAL=8084
+		VCELL_S3PROXY_PORT=8106
 		VCELL_DEBUG_PORT_BASE=5060
 		_applicationId="1471-8022-1038-5558"
 		;;
 	TEST5)
 		_site_port_offset=7
 		VCELL_API_HOST_EXTERNAL=vcellapi-beta.cam.uchc.edu
 		VCELL_API_PORT_EXTERNAL=8085
+		VCELL_S3PROXY_PORT=8107
 		VCELL_DEBUG_PORT_BASE=5070
 		_applicationId="1471-8022-1038-5559"
 		;;
@@ -200,6 +208,8 @@ VCELL_OPT_DOCKER_IMAGE=$VCELL_OPT_DOCKER_IMAGE
 VCELL_OPT_SINGULARITY_FILENAME=$VCELL_OPT_SINGULARITY_FILENAME
 VCELL_OPT_SINGULARITY_IMAGE_EXTERNAL=$VCELL_OPT_SINGULARITY_IMAGE_EXTERNAL
 VCELL_REPO_NAMESPACE=$VCELL_REPO_NAMESPACE
+VCELL_S3PROXY_DIR=/share/apps/vcell3/s3proxy
+VCELL_S3PROXY_PORT=$VCELL_S3PROXY_PORT
 VCELL_SECRETS_DIR=/usr/local/deploy
 VCELL_SIMDATADIR_ARCHIVE_EXTERNAL=/share/apps/vcell12/users
 VCELL_SIMDATADIR_ARCHIVE_INTERNAL=/share/apps/vcell12/users

diff --git a/vcell-core/src/main/java/cbit/vcell/simdata/n5/N5Exporter.java b/vcell-core/src/main/java/cbit/vcell/simdata/n5/N5Exporter.java
@@ -0,0 +1,48 @@
+package cbit.vcell.simdata.n5;
+
+import cbit.vcell.export.server.ExportConstants;
+import cbit.vcell.resource.PropertyLoader;
+import cbit.vcell.simdata.*;
+import cbit.vcell.solver.VCSimulationDataIdentifier;
+import cbit.vcell.solver.VCSimulationIdentifier;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.vcell.util.document.KeyValue;
+import org.vcell.util.document.User;
+
+import java.io.File;
+import java.util.Arrays;
+
+public class N5Exporter implements ExportConstants {
+    private final static Logger lg = LogManager.getLogger(N5Exporter.class);
+
+    public N5Exporter() {
+
+    }
+
+    public static void main(String[] args) throws Exception {
+        String primaryDirStr = "~/.vcell/simdata/temp";
+        PropertyLoader.loadProperties();
+        //System.setProperty(PropertyLoader.installationRoot,"/Users/schaff/workspace/vcell");
+        User user = new User("schaff", new KeyValue("17"));
+        KeyValue simKey = new KeyValue("1128909032");
+        VCSimulationIdentifier vcSimID = new VCSimulationIdentifier(simKey, user);
+        VCSimulationDataIdentifier vcdID = new VCSimulationDataIdentifier(vcSimID, 0);
+
+        Cachetable cachetable = new Cachetable(10 * Cachetable.minute, 1000000L);
+        File primaryDir = new File(primaryDirStr);
+        DataSetControllerImpl dataSetControllerImpl = new DataSetControllerImpl(cachetable, primaryDir, null);
+        double[] allTimes = dataSetControllerImpl.getDataSetTimes(vcdID);
+
+        // get dataset identifier for vcSimID
+        VCData vcData = dataSetControllerImpl.getVCData(vcdID);
+        OutputContext outputContext = new OutputContext(null);
+        DataIdentifier[] dataIdentifiers = vcData.getVarAndFunctionDataIdentifiers(outputContext);
+        DataIdentifier DexDataIdentifier = Arrays.stream(dataIdentifiers).filter(di -> di.getName().equals("Dex")).findFirst().get();
+        SimDataBlock simDataBlock = vcData.getSimDataBlock(outputContext, DexDataIdentifier.getName(), allTimes[0]);
+        double[] Dex_dataBlockValues = simDataBlock.getData(); // X,Y,Z raster for time 0 for variable cAMP
+        System.out.println("Dex_dataBlockValues.length = " + Dex_dataBlockValues.length);
+        System.out.println("Dex_dataBlockValues[0] = " + Dex_dataBlockValues[0]);
+
+    }
+}
diff --git a/vcell-server/pom.xml b/vcell-server/pom.xml
@@ -139,5 +139,12 @@
 			<version>42.4.1</version>
 		</dependency>
 
+<!--		For S3Proxy package -->
+		<dependency>
+			<groupId>org.gaul</groupId>
+			<artifactId>s3proxy</artifactId>
+			<version>1.7.1</version>
+		</dependency>
+
 	</dependencies>
 </project>
diff --git a/vcell-server/src/main/java/org/vcell/s3/S3ProxyVCell.java b/vcell-server/src/main/java/org/vcell/s3/S3ProxyVCell.java
@@ -0,0 +1,44 @@
+package org.vcell.s3;
+
+import org.gaul.s3proxy.S3Proxy;
+import org.gaul.shaded.org.eclipse.jetty.util.component.AbstractLifeCycle;
+import org.jclouds.ContextBuilder;
+import org.jclouds.blobstore.BlobStore;
+import org.jclouds.blobstore.BlobStoreContext;
+import org.jclouds.blobstore.config.LocalBlobStore;
+import org.jclouds.filesystem.reference.FilesystemConstants;
+
+import java.net.URI;
+import java.util.Properties;
+
+public class S3ProxyVCell {
+
+    public static void main(String[] args) {
+        Properties properties = new Properties();
+        String tmp = "/media/zeke/DiskDrive/Home/Work/CCAM/TempStorage";
+        properties.setProperty(FilesystemConstants.PROPERTY_BASEDIR, tmp);
+
+        BlobStore blobStore;
+
+//        BlobStoreContext localBlobStore = ContextBuilder.newBuilder("filesystem").overrides(properties).build(BlobStoreContext.class);
+
+        BlobStoreContext blobStoreContext = ContextBuilder
+                .newBuilder("filesystem")
+                .overrides(properties)
+                .buildView(BlobStoreContext.class);
+        // error
+//        https://jclouds.apache.org/reference/javadoc/2.0.x/org/jclouds/blobstore/BlobStore.html
+
+        S3Proxy s3Proxy = S3Proxy.builder()
+                .blobStore(blobStoreContext.getBlobStore())
+                .endpoint(URI.create("http://127.0.0.1:8080"))
+                .servicePath(tmp)
+                .build();
+
+        try {
+            s3Proxy.start();
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+}