diff --git a/docker/swarm/docker-compose.yml b/docker/swarm/docker-compose.yml index e177710200..c8bdfd6e0d 100644 --- a/docker/swarm/docker-compose.yml +++ b/docker/swarm/docker-compose.yml @@ -349,6 +349,56 @@ services: placement: constraints: - node.labels.zone == INTERNAL + + s3proxy: + image: andrewgaul/s3proxy:latest + ports: + - "${VCELL_S3PROXY_PORT}:80" +# - "${VCELL_S3PROXY_PORT}:443" + environment: + - LOG_LEVEL="info" + - S3PROXY_AUTHORIZATION="none" #Set it to None means that anyone can access and use this +# - S3PROXY_ENDPOINT="http://0.0.0.0:80" +# - S3PROXY_IDENTITY="local-identity" +# - S3PROXY_CREDENTIAL="local-credential" +# - S3PROXY_VIRTUALHOST="" +# - S3PROXY_KEYSTORE_PATH="keystore.jks" +# - S3PROXY_KEYSTORE_PASSWORD="password" + - S3PROXY_CORS_ALLOW_ALL="false" +# - S3PROXY_CORS_ALLOW_ORIGINS="" +# - S3PROXY_CORS_ALLOW_METHODS="" +# - S3PROXY_CORS_ALLOW_HEADERS="" + - S3PROXY_IGNORE_UNKNOWN_HEADERS="false" +# - S3PROXY_ENCRYPTED_BLOBSTORE="" +# - S3PROXY_ENCRYPTED_BLOBSTORE_PASSWORD="" +# - S3PROXY_ENCRYPTED_BLOBSTORE_SALT="" + - JCLOUDS_PROVIDER="filesystem" +# - JCLOUDS_ENDPOINT="" +# - JCLOUDS_REGION="" +# - JCLOUDS_REGIONS="us-east-1" +# - JCLOUDS_IDENTITY="remote-identity" +# - JCLOUDS_CREDENTIAL="remote-credential" +# - JCLOUDS_KEYSTONE_VERSION="" +# - JCLOUDS_KEYSTONE_SCOPE="" +# - JCLOUDS_KEYSTONE_PROJECT_DOMAIN_NAME="" + - JCLOUDS_FILESYSTEM_BASEDIR="/data" + volumes: + - "${VCELL_S3PROXY_DIR}:/data" + networks: + - vcellnet + deploy: + mode: replicated + replicas: 1 + resources: + limits: + memory: 2000M + reservations: + memory: 1000M + placement: + constraints: + - node.labels.zone == INTERNAL + + mongodb: image: "${VCELL_REPO_NAMESPACE}/vcell-mongo:${VCELL_TAG}" ports: diff --git a/docker/swarm/localconfig_realslurm_oracle.sh b/docker/swarm/localconfig_realslurm_oracle.sh index 150cac993a..dbb47157f5 100755 --- a/docker/swarm/localconfig_realslurm_oracle.sh +++ b/docker/swarm/localconfig_realslurm_oracle.sh @@ -105,6 +105,7 @@ VCELL_SLURM_CENTRAL_SINGULARITY_DIR=/share/apps/vcell3/singularityImages # VCELL_API_PORT_EXTERNAL=$((8080 + _site_port_offset)) +VCELL_S3PROXY_PORT=$((8100 + _site_port_offset)) VCELL_JMS_SIM_PORT_EXTERNAL=$((61616 + _site_port_offset)) VCELL_JMS_SIM_RESTPORT_EXTERNAL=$((8161 + _site_port_offset)) VCELL_MONGO_PORT_EXTERNAL=$((27017 + _site_port_offset)) @@ -178,6 +179,8 @@ VCELL_OPT_DOCKER_IMAGE=$VCELL_OPT_DOCKER_IMAGE VCELL_OPT_SINGULARITY_FILENAME=$VCELL_OPT_SINGULARITY_FILENAME VCELL_OPT_SINGULARITY_IMAGE_EXTERNAL=$VCELL_OPT_SINGULARITY_IMAGE_EXTERNAL VCELL_REPO_NAMESPACE=$VCELL_REPO_NAMESPACE +VCELL_S3PROXY_DIR=/Volumes/vcell/s3proxy +VCELL_S3PROXY_PORT=$VCELL_S3PROXY_PORT VCELL_SECRETS_DIR=${HOME}/vcellkeys_oracle VCELL_SIMDATADIR_ARCHIVE_EXTERNAL=/share/apps/vcell12/users VCELL_SIMDATADIR_ARCHIVE_INTERNAL=/share/apps/vcell12/users diff --git a/docker/swarm/localconfig_realslurm_postgres.sh b/docker/swarm/localconfig_realslurm_postgres.sh index 11d1455aaf..900e07c0d1 100755 --- a/docker/swarm/localconfig_realslurm_postgres.sh +++ b/docker/swarm/localconfig_realslurm_postgres.sh @@ -105,6 +105,7 @@ VCELL_SLURM_CENTRAL_SINGULARITY_DIR=/share/apps/vcell3/singularityImages # VCELL_API_PORT_EXTERNAL=$((8080 + _site_port_offset)) +VCELL_S3PROXY_PORT=$((8100 + _site_port_offset)) VCELL_JMS_SIM_PORT_EXTERNAL=$((61616 + _site_port_offset)) VCELL_JMS_SIM_RESTPORT_EXTERNAL=$((8161 + _site_port_offset)) VCELL_MONGO_PORT_EXTERNAL=$((27017 + _site_port_offset)) @@ -178,6 +179,8 @@ VCELL_OPT_DOCKER_IMAGE=$VCELL_OPT_DOCKER_IMAGE VCELL_OPT_SINGULARITY_FILENAME=$VCELL_OPT_SINGULARITY_FILENAME VCELL_OPT_SINGULARITY_IMAGE_EXTERNAL=$VCELL_OPT_SINGULARITY_IMAGE_EXTERNAL VCELL_REPO_NAMESPACE=$VCELL_REPO_NAMESPACE +VCELL_S3PROXY_DIR=/Volumes/vcell/s3proxy +VCELL_S3PROXY_PORT=$VCELL_S3PROXY_PORT VCELL_SECRETS_DIR=${HOME}/vcellkeys_postgres VCELL_SIMDATADIR_ARCHIVE_EXTERNAL=/share/apps/vcell12/users VCELL_SIMDATADIR_ARCHIVE_INTERNAL=/share/apps/vcell12/users diff --git a/docker/swarm/serverconfig-uch.sh b/docker/swarm/serverconfig-uch.sh index 3a96975aa2..e11e611201 100755 --- a/docker/swarm/serverconfig-uch.sh +++ b/docker/swarm/serverconfig-uch.sh @@ -40,6 +40,7 @@ case $VCELL_SITE in _site_port_offset=0 VCELL_API_HOST_EXTERNAL=vcellapi.cam.uchc.edu VCELL_API_PORT_EXTERNAL=443 + VCELL_S3PROXY_PORT=8100 VCELL_DEBUG_PORT_BASE=5000 _applicationId="1471-8022-1038-5553" ;; @@ -47,6 +48,7 @@ case $VCELL_SITE in _site_port_offset=1 VCELL_API_HOST_EXTERNAL=vcellapi.cam.uchc.edu VCELL_API_PORT_EXTERNAL=443 + VCELL_S3PROXY_PORT=8101 VCELL_DEBUG_PORT_BASE=5010 _applicationId="1471-8022-1038-5552" ;; @@ -54,6 +56,7 @@ case $VCELL_SITE in _site_port_offset=2 VCELL_API_HOST_EXTERNAL=vcellapi-beta.cam.uchc.edu VCELL_API_PORT_EXTERNAL=8080 + VCELL_S3PROXY_PORT=8102 VCELL_DEBUG_PORT_BASE=5020 _applicationId="1471-8022-1038-5554" ;; @@ -62,6 +65,7 @@ case $VCELL_SITE in # VCELL_API_PORT_EXTERNAL=8081 VCELL_API_HOST_EXTERNAL=vcellapi-beta.cam.uchc.edu VCELL_API_PORT_EXTERNAL=443 + VCELL_S3PROXY_PORT=8103 VCELL_DEBUG_PORT_BASE=5030 _applicationId="1471-8022-1038-5555" ;; @@ -69,6 +73,7 @@ case $VCELL_SITE in _site_port_offset=4 VCELL_API_HOST_EXTERNAL=vcellapi-beta.cam.uchc.edu VCELL_API_PORT_EXTERNAL=8082 + VCELL_S3PROXY_PORT=8104 VCELL_DEBUG_PORT_BASE=5040 _applicationId="1471-8022-1038-5556" ;; @@ -76,6 +81,7 @@ case $VCELL_SITE in _site_port_offset=5 VCELL_API_HOST_EXTERNAL=vcellapi-beta.cam.uchc.edu VCELL_API_PORT_EXTERNAL=8083 + VCELL_S3PROXY_PORT=8105 VCELL_DEBUG_PORT_BASE=5050 _applicationId="1471-8022-1038-5557" ;; @@ -83,6 +89,7 @@ case $VCELL_SITE in _site_port_offset=6 VCELL_API_HOST_EXTERNAL=vcellapi-beta.cam.uchc.edu VCELL_API_PORT_EXTERNAL=8084 + VCELL_S3PROXY_PORT=8106 VCELL_DEBUG_PORT_BASE=5060 _applicationId="1471-8022-1038-5558" ;; @@ -90,6 +97,7 @@ case $VCELL_SITE in _site_port_offset=7 VCELL_API_HOST_EXTERNAL=vcellapi-beta.cam.uchc.edu VCELL_API_PORT_EXTERNAL=8085 + VCELL_S3PROXY_PORT=8107 VCELL_DEBUG_PORT_BASE=5070 _applicationId="1471-8022-1038-5559" ;; @@ -200,6 +208,8 @@ VCELL_OPT_DOCKER_IMAGE=$VCELL_OPT_DOCKER_IMAGE VCELL_OPT_SINGULARITY_FILENAME=$VCELL_OPT_SINGULARITY_FILENAME VCELL_OPT_SINGULARITY_IMAGE_EXTERNAL=$VCELL_OPT_SINGULARITY_IMAGE_EXTERNAL VCELL_REPO_NAMESPACE=$VCELL_REPO_NAMESPACE +VCELL_S3PROXY_DIR=/share/apps/vcell3/s3proxy +VCELL_S3PROXY_PORT=$VCELL_S3PROXY_PORT VCELL_SECRETS_DIR=/usr/local/deploy VCELL_SIMDATADIR_ARCHIVE_EXTERNAL=/share/apps/vcell12/users VCELL_SIMDATADIR_ARCHIVE_INTERNAL=/share/apps/vcell12/users diff --git a/vcell-core/src/main/java/cbit/vcell/simdata/n5/N5Exporter.java b/vcell-core/src/main/java/cbit/vcell/simdata/n5/N5Exporter.java new file mode 100644 index 0000000000..ab0f565b1b --- /dev/null +++ b/vcell-core/src/main/java/cbit/vcell/simdata/n5/N5Exporter.java @@ -0,0 +1,48 @@ +package cbit.vcell.simdata.n5; + +import cbit.vcell.export.server.ExportConstants; +import cbit.vcell.resource.PropertyLoader; +import cbit.vcell.simdata.*; +import cbit.vcell.solver.VCSimulationDataIdentifier; +import cbit.vcell.solver.VCSimulationIdentifier; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vcell.util.document.KeyValue; +import org.vcell.util.document.User; + +import java.io.File; +import java.util.Arrays; + +public class N5Exporter implements ExportConstants { + private final static Logger lg = LogManager.getLogger(N5Exporter.class); + + public N5Exporter() { + + } + + public static void main(String[] args) throws Exception { + String primaryDirStr = "~/.vcell/simdata/temp"; + PropertyLoader.loadProperties(); + //System.setProperty(PropertyLoader.installationRoot,"/Users/schaff/workspace/vcell"); + User user = new User("schaff", new KeyValue("17")); + KeyValue simKey = new KeyValue("1128909032"); + VCSimulationIdentifier vcSimID = new VCSimulationIdentifier(simKey, user); + VCSimulationDataIdentifier vcdID = new VCSimulationDataIdentifier(vcSimID, 0); + + Cachetable cachetable = new Cachetable(10 * Cachetable.minute, 1000000L); + File primaryDir = new File(primaryDirStr); + DataSetControllerImpl dataSetControllerImpl = new DataSetControllerImpl(cachetable, primaryDir, null); + double[] allTimes = dataSetControllerImpl.getDataSetTimes(vcdID); + + // get dataset identifier for vcSimID + VCData vcData = dataSetControllerImpl.getVCData(vcdID); + OutputContext outputContext = new OutputContext(null); + DataIdentifier[] dataIdentifiers = vcData.getVarAndFunctionDataIdentifiers(outputContext); + DataIdentifier DexDataIdentifier = Arrays.stream(dataIdentifiers).filter(di -> di.getName().equals("Dex")).findFirst().get(); + SimDataBlock simDataBlock = vcData.getSimDataBlock(outputContext, DexDataIdentifier.getName(), allTimes[0]); + double[] Dex_dataBlockValues = simDataBlock.getData(); // X,Y,Z raster for time 0 for variable cAMP + System.out.println("Dex_dataBlockValues.length = " + Dex_dataBlockValues.length); + System.out.println("Dex_dataBlockValues[0] = " + Dex_dataBlockValues[0]); + + } +} \ No newline at end of file diff --git a/vcell-server/pom.xml b/vcell-server/pom.xml index 77d748d5bf..abe3f04ead 100644 --- a/vcell-server/pom.xml +++ b/vcell-server/pom.xml @@ -139,5 +139,12 @@ 42.4.1 + + + org.gaul + s3proxy + 1.7.1 + + diff --git a/vcell-server/src/main/java/org/vcell/s3/S3ProxyVCell.java b/vcell-server/src/main/java/org/vcell/s3/S3ProxyVCell.java new file mode 100644 index 0000000000..d66bd6f407 --- /dev/null +++ b/vcell-server/src/main/java/org/vcell/s3/S3ProxyVCell.java @@ -0,0 +1,44 @@ +package org.vcell.s3; + +import org.gaul.s3proxy.S3Proxy; +import org.gaul.shaded.org.eclipse.jetty.util.component.AbstractLifeCycle; +import org.jclouds.ContextBuilder; +import org.jclouds.blobstore.BlobStore; +import org.jclouds.blobstore.BlobStoreContext; +import org.jclouds.blobstore.config.LocalBlobStore; +import org.jclouds.filesystem.reference.FilesystemConstants; + +import java.net.URI; +import java.util.Properties; + +public class S3ProxyVCell { + + public static void main(String[] args) { + Properties properties = new Properties(); + String tmp = "/media/zeke/DiskDrive/Home/Work/CCAM/TempStorage"; + properties.setProperty(FilesystemConstants.PROPERTY_BASEDIR, tmp); + + BlobStore blobStore; + +// BlobStoreContext localBlobStore = ContextBuilder.newBuilder("filesystem").overrides(properties).build(BlobStoreContext.class); + + BlobStoreContext blobStoreContext = ContextBuilder + .newBuilder("filesystem") + .overrides(properties) + .buildView(BlobStoreContext.class); + // error +// https://jclouds.apache.org/reference/javadoc/2.0.x/org/jclouds/blobstore/BlobStore.html + + S3Proxy s3Proxy = S3Proxy.builder() + .blobStore(blobStoreContext.getBlobStore()) + .endpoint(URI.create("http://127.0.0.1:8080")) + .servicePath(tmp) + .build(); + + try { + s3Proxy.start(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +}