From 54a50610400b88ead70b14eb36b185ce1b12fb84 Mon Sep 17 00:00:00 2001 From: Jim Schaff Date: Mon, 21 Aug 2023 11:42:34 -0400 Subject: [PATCH 1/4] initial S3Proxy server configuration --- docker/swarm/docker-compose.yml | 50 +++++++++++++++++++ docker/swarm/localconfig_realslurm_oracle.sh | 3 ++ .../swarm/localconfig_realslurm_postgres.sh | 3 ++ docker/swarm/serverconfig-uch.sh | 10 ++++ 4 files changed, 66 insertions(+) diff --git a/docker/swarm/docker-compose.yml b/docker/swarm/docker-compose.yml index e177710200..8a1d56a952 100644 --- a/docker/swarm/docker-compose.yml +++ b/docker/swarm/docker-compose.yml @@ -349,6 +349,56 @@ services: placement: constraints: - node.labels.zone == INTERNAL + + s3proxy: + image: andrewgaul/s3proxy:latest + ports: + - "${VCELL_S3PROXY_PORT}:80" +# - "${VCELL_S3PROXY_PORT}:443" + environment: + - LOG_LEVEL="info" +# - S3PROXY_AUTHORIZATION="aws-v2-or-v4" +# - S3PROXY_ENDPOINT="http://0.0.0.0:80" +# - S3PROXY_IDENTITY="local-identity" +# - S3PROXY_CREDENTIAL="local-credential" +# - S3PROXY_VIRTUALHOST="" +# - S3PROXY_KEYSTORE_PATH="keystore.jks" +# - S3PROXY_KEYSTORE_PASSWORD="password" + - S3PROXY_CORS_ALLOW_ALL="true" +# - S3PROXY_CORS_ALLOW_ORIGINS="" +# - S3PROXY_CORS_ALLOW_METHODS="" +# - S3PROXY_CORS_ALLOW_HEADERS="" + - S3PROXY_IGNORE_UNKNOWN_HEADERS="true" +# - S3PROXY_ENCRYPTED_BLOBSTORE="" +# - S3PROXY_ENCRYPTED_BLOBSTORE_PASSWORD="" +# - S3PROXY_ENCRYPTED_BLOBSTORE_SALT="" + - JCLOUDS_PROVIDER="filesystem" +# - JCLOUDS_ENDPOINT="" +# - JCLOUDS_REGION="" +# - JCLOUDS_REGIONS="us-east-1" +# - JCLOUDS_IDENTITY="remote-identity" +# - JCLOUDS_CREDENTIAL="remote-credential" +# - JCLOUDS_KEYSTONE_VERSION="" +# - JCLOUDS_KEYSTONE_SCOPE="" +# - JCLOUDS_KEYSTONE_PROJECT_DOMAIN_NAME="" + - JCLOUDS_FILESYSTEM_BASEDIR="/data" + volumes: + - "${VCELL_S3PROXY_DIR}:/data" + networks: + - vcellnet + deploy: + mode: replicated + replicas: 1 + resources: + limits: + memory: 2000M + reservations: + memory: 1000M + placement: + constraints: + - node.labels.zone == INTERNAL + + mongodb: image: "${VCELL_REPO_NAMESPACE}/vcell-mongo:${VCELL_TAG}" ports: diff --git a/docker/swarm/localconfig_realslurm_oracle.sh b/docker/swarm/localconfig_realslurm_oracle.sh index 150cac993a..dbb47157f5 100755 --- a/docker/swarm/localconfig_realslurm_oracle.sh +++ b/docker/swarm/localconfig_realslurm_oracle.sh @@ -105,6 +105,7 @@ VCELL_SLURM_CENTRAL_SINGULARITY_DIR=/share/apps/vcell3/singularityImages # VCELL_API_PORT_EXTERNAL=$((8080 + _site_port_offset)) +VCELL_S3PROXY_PORT=$((8100 + _site_port_offset)) VCELL_JMS_SIM_PORT_EXTERNAL=$((61616 + _site_port_offset)) VCELL_JMS_SIM_RESTPORT_EXTERNAL=$((8161 + _site_port_offset)) VCELL_MONGO_PORT_EXTERNAL=$((27017 + _site_port_offset)) @@ -178,6 +179,8 @@ VCELL_OPT_DOCKER_IMAGE=$VCELL_OPT_DOCKER_IMAGE VCELL_OPT_SINGULARITY_FILENAME=$VCELL_OPT_SINGULARITY_FILENAME VCELL_OPT_SINGULARITY_IMAGE_EXTERNAL=$VCELL_OPT_SINGULARITY_IMAGE_EXTERNAL VCELL_REPO_NAMESPACE=$VCELL_REPO_NAMESPACE +VCELL_S3PROXY_DIR=/Volumes/vcell/s3proxy +VCELL_S3PROXY_PORT=$VCELL_S3PROXY_PORT VCELL_SECRETS_DIR=${HOME}/vcellkeys_oracle VCELL_SIMDATADIR_ARCHIVE_EXTERNAL=/share/apps/vcell12/users VCELL_SIMDATADIR_ARCHIVE_INTERNAL=/share/apps/vcell12/users diff --git a/docker/swarm/localconfig_realslurm_postgres.sh b/docker/swarm/localconfig_realslurm_postgres.sh index 11d1455aaf..900e07c0d1 100755 --- a/docker/swarm/localconfig_realslurm_postgres.sh +++ b/docker/swarm/localconfig_realslurm_postgres.sh @@ -105,6 +105,7 @@ VCELL_SLURM_CENTRAL_SINGULARITY_DIR=/share/apps/vcell3/singularityImages # VCELL_API_PORT_EXTERNAL=$((8080 + _site_port_offset)) +VCELL_S3PROXY_PORT=$((8100 + _site_port_offset)) VCELL_JMS_SIM_PORT_EXTERNAL=$((61616 + _site_port_offset)) VCELL_JMS_SIM_RESTPORT_EXTERNAL=$((8161 + _site_port_offset)) VCELL_MONGO_PORT_EXTERNAL=$((27017 + _site_port_offset)) @@ -178,6 +179,8 @@ VCELL_OPT_DOCKER_IMAGE=$VCELL_OPT_DOCKER_IMAGE VCELL_OPT_SINGULARITY_FILENAME=$VCELL_OPT_SINGULARITY_FILENAME VCELL_OPT_SINGULARITY_IMAGE_EXTERNAL=$VCELL_OPT_SINGULARITY_IMAGE_EXTERNAL VCELL_REPO_NAMESPACE=$VCELL_REPO_NAMESPACE +VCELL_S3PROXY_DIR=/Volumes/vcell/s3proxy +VCELL_S3PROXY_PORT=$VCELL_S3PROXY_PORT VCELL_SECRETS_DIR=${HOME}/vcellkeys_postgres VCELL_SIMDATADIR_ARCHIVE_EXTERNAL=/share/apps/vcell12/users VCELL_SIMDATADIR_ARCHIVE_INTERNAL=/share/apps/vcell12/users diff --git a/docker/swarm/serverconfig-uch.sh b/docker/swarm/serverconfig-uch.sh index 3a96975aa2..e11e611201 100755 --- a/docker/swarm/serverconfig-uch.sh +++ b/docker/swarm/serverconfig-uch.sh @@ -40,6 +40,7 @@ case $VCELL_SITE in _site_port_offset=0 VCELL_API_HOST_EXTERNAL=vcellapi.cam.uchc.edu VCELL_API_PORT_EXTERNAL=443 + VCELL_S3PROXY_PORT=8100 VCELL_DEBUG_PORT_BASE=5000 _applicationId="1471-8022-1038-5553" ;; @@ -47,6 +48,7 @@ case $VCELL_SITE in _site_port_offset=1 VCELL_API_HOST_EXTERNAL=vcellapi.cam.uchc.edu VCELL_API_PORT_EXTERNAL=443 + VCELL_S3PROXY_PORT=8101 VCELL_DEBUG_PORT_BASE=5010 _applicationId="1471-8022-1038-5552" ;; @@ -54,6 +56,7 @@ case $VCELL_SITE in _site_port_offset=2 VCELL_API_HOST_EXTERNAL=vcellapi-beta.cam.uchc.edu VCELL_API_PORT_EXTERNAL=8080 + VCELL_S3PROXY_PORT=8102 VCELL_DEBUG_PORT_BASE=5020 _applicationId="1471-8022-1038-5554" ;; @@ -62,6 +65,7 @@ case $VCELL_SITE in # VCELL_API_PORT_EXTERNAL=8081 VCELL_API_HOST_EXTERNAL=vcellapi-beta.cam.uchc.edu VCELL_API_PORT_EXTERNAL=443 + VCELL_S3PROXY_PORT=8103 VCELL_DEBUG_PORT_BASE=5030 _applicationId="1471-8022-1038-5555" ;; @@ -69,6 +73,7 @@ case $VCELL_SITE in _site_port_offset=4 VCELL_API_HOST_EXTERNAL=vcellapi-beta.cam.uchc.edu VCELL_API_PORT_EXTERNAL=8082 + VCELL_S3PROXY_PORT=8104 VCELL_DEBUG_PORT_BASE=5040 _applicationId="1471-8022-1038-5556" ;; @@ -76,6 +81,7 @@ case $VCELL_SITE in _site_port_offset=5 VCELL_API_HOST_EXTERNAL=vcellapi-beta.cam.uchc.edu VCELL_API_PORT_EXTERNAL=8083 + VCELL_S3PROXY_PORT=8105 VCELL_DEBUG_PORT_BASE=5050 _applicationId="1471-8022-1038-5557" ;; @@ -83,6 +89,7 @@ case $VCELL_SITE in _site_port_offset=6 VCELL_API_HOST_EXTERNAL=vcellapi-beta.cam.uchc.edu VCELL_API_PORT_EXTERNAL=8084 + VCELL_S3PROXY_PORT=8106 VCELL_DEBUG_PORT_BASE=5060 _applicationId="1471-8022-1038-5558" ;; @@ -90,6 +97,7 @@ case $VCELL_SITE in _site_port_offset=7 VCELL_API_HOST_EXTERNAL=vcellapi-beta.cam.uchc.edu VCELL_API_PORT_EXTERNAL=8085 + VCELL_S3PROXY_PORT=8107 VCELL_DEBUG_PORT_BASE=5070 _applicationId="1471-8022-1038-5559" ;; @@ -200,6 +208,8 @@ VCELL_OPT_DOCKER_IMAGE=$VCELL_OPT_DOCKER_IMAGE VCELL_OPT_SINGULARITY_FILENAME=$VCELL_OPT_SINGULARITY_FILENAME VCELL_OPT_SINGULARITY_IMAGE_EXTERNAL=$VCELL_OPT_SINGULARITY_IMAGE_EXTERNAL VCELL_REPO_NAMESPACE=$VCELL_REPO_NAMESPACE +VCELL_S3PROXY_DIR=/share/apps/vcell3/s3proxy +VCELL_S3PROXY_PORT=$VCELL_S3PROXY_PORT VCELL_SECRETS_DIR=/usr/local/deploy VCELL_SIMDATADIR_ARCHIVE_EXTERNAL=/share/apps/vcell12/users VCELL_SIMDATADIR_ARCHIVE_INTERNAL=/share/apps/vcell12/users From d266496fc4a2742eca8ad9b458f40aa18baaade5 Mon Sep 17 00:00:00 2001 From: Ezequiel Valencia Date: Tue, 22 Aug 2023 13:37:32 -0400 Subject: [PATCH 2/4] Modifying S3Proxy Environmental Variables Made it so that there is no CORS requests available, allowing only known headers, and require no authorization. There shouldn't be any other domains trying to access the resources in the S3 buckets since we'll me accessing it directly. If we want authorization then we need to make an AWS account which is responsible for this. Unknown headers should not be used since we'll be using already made API tools (AWS libraries/CLI) and they shouldn't act funny. --- docker/swarm/docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/swarm/docker-compose.yml b/docker/swarm/docker-compose.yml index 8a1d56a952..c8bdfd6e0d 100644 --- a/docker/swarm/docker-compose.yml +++ b/docker/swarm/docker-compose.yml @@ -357,18 +357,18 @@ services: # - "${VCELL_S3PROXY_PORT}:443" environment: - LOG_LEVEL="info" -# - S3PROXY_AUTHORIZATION="aws-v2-or-v4" + - S3PROXY_AUTHORIZATION="none" #Set it to None means that anyone can access and use this # - S3PROXY_ENDPOINT="http://0.0.0.0:80" # - S3PROXY_IDENTITY="local-identity" # - S3PROXY_CREDENTIAL="local-credential" # - S3PROXY_VIRTUALHOST="" # - S3PROXY_KEYSTORE_PATH="keystore.jks" # - S3PROXY_KEYSTORE_PASSWORD="password" - - S3PROXY_CORS_ALLOW_ALL="true" + - S3PROXY_CORS_ALLOW_ALL="false" # - S3PROXY_CORS_ALLOW_ORIGINS="" # - S3PROXY_CORS_ALLOW_METHODS="" # - S3PROXY_CORS_ALLOW_HEADERS="" - - S3PROXY_IGNORE_UNKNOWN_HEADERS="true" + - S3PROXY_IGNORE_UNKNOWN_HEADERS="false" # - S3PROXY_ENCRYPTED_BLOBSTORE="" # - S3PROXY_ENCRYPTED_BLOBSTORE_PASSWORD="" # - S3PROXY_ENCRYPTED_BLOBSTORE_SALT="" From ab104f3a5fada8e0f83e2e3195deb2a46502201d Mon Sep 17 00:00:00 2001 From: Ezequiel Valencia Date: Wed, 23 Aug 2023 10:42:54 -0400 Subject: [PATCH 3/4] S3Proxy JRE Server Made a mockup of S3Proxy in a JRE. --- vcell-server/pom.xml | 7 +++ .../main/java/org/vcell/s3/S3ProxyVCell.java | 44 +++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 vcell-server/src/main/java/org/vcell/s3/S3ProxyVCell.java diff --git a/vcell-server/pom.xml b/vcell-server/pom.xml index 77d748d5bf..abe3f04ead 100644 --- a/vcell-server/pom.xml +++ b/vcell-server/pom.xml @@ -139,5 +139,12 @@ 42.4.1 + + + org.gaul + s3proxy + 1.7.1 + + diff --git a/vcell-server/src/main/java/org/vcell/s3/S3ProxyVCell.java b/vcell-server/src/main/java/org/vcell/s3/S3ProxyVCell.java new file mode 100644 index 0000000000..d66bd6f407 --- /dev/null +++ b/vcell-server/src/main/java/org/vcell/s3/S3ProxyVCell.java @@ -0,0 +1,44 @@ +package org.vcell.s3; + +import org.gaul.s3proxy.S3Proxy; +import org.gaul.shaded.org.eclipse.jetty.util.component.AbstractLifeCycle; +import org.jclouds.ContextBuilder; +import org.jclouds.blobstore.BlobStore; +import org.jclouds.blobstore.BlobStoreContext; +import org.jclouds.blobstore.config.LocalBlobStore; +import org.jclouds.filesystem.reference.FilesystemConstants; + +import java.net.URI; +import java.util.Properties; + +public class S3ProxyVCell { + + public static void main(String[] args) { + Properties properties = new Properties(); + String tmp = "/media/zeke/DiskDrive/Home/Work/CCAM/TempStorage"; + properties.setProperty(FilesystemConstants.PROPERTY_BASEDIR, tmp); + + BlobStore blobStore; + +// BlobStoreContext localBlobStore = ContextBuilder.newBuilder("filesystem").overrides(properties).build(BlobStoreContext.class); + + BlobStoreContext blobStoreContext = ContextBuilder + .newBuilder("filesystem") + .overrides(properties) + .buildView(BlobStoreContext.class); + // error +// https://jclouds.apache.org/reference/javadoc/2.0.x/org/jclouds/blobstore/BlobStore.html + + S3Proxy s3Proxy = S3Proxy.builder() + .blobStore(blobStoreContext.getBlobStore()) + .endpoint(URI.create("http://127.0.0.1:8080")) + .servicePath(tmp) + .build(); + + try { + s3Proxy.start(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} From 997a96fae4797d548c0cab639d8b6195e71017e8 Mon Sep 17 00:00:00 2001 From: Jim Schaff Date: Wed, 23 Aug 2023 12:26:50 -0400 Subject: [PATCH 4/4] WIP: N5Exporter - first look --- .../cbit/vcell/simdata/n5/N5Exporter.java | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 vcell-core/src/main/java/cbit/vcell/simdata/n5/N5Exporter.java diff --git a/vcell-core/src/main/java/cbit/vcell/simdata/n5/N5Exporter.java b/vcell-core/src/main/java/cbit/vcell/simdata/n5/N5Exporter.java new file mode 100644 index 0000000000..ab0f565b1b --- /dev/null +++ b/vcell-core/src/main/java/cbit/vcell/simdata/n5/N5Exporter.java @@ -0,0 +1,48 @@ +package cbit.vcell.simdata.n5; + +import cbit.vcell.export.server.ExportConstants; +import cbit.vcell.resource.PropertyLoader; +import cbit.vcell.simdata.*; +import cbit.vcell.solver.VCSimulationDataIdentifier; +import cbit.vcell.solver.VCSimulationIdentifier; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vcell.util.document.KeyValue; +import org.vcell.util.document.User; + +import java.io.File; +import java.util.Arrays; + +public class N5Exporter implements ExportConstants { + private final static Logger lg = LogManager.getLogger(N5Exporter.class); + + public N5Exporter() { + + } + + public static void main(String[] args) throws Exception { + String primaryDirStr = "~/.vcell/simdata/temp"; + PropertyLoader.loadProperties(); + //System.setProperty(PropertyLoader.installationRoot,"/Users/schaff/workspace/vcell"); + User user = new User("schaff", new KeyValue("17")); + KeyValue simKey = new KeyValue("1128909032"); + VCSimulationIdentifier vcSimID = new VCSimulationIdentifier(simKey, user); + VCSimulationDataIdentifier vcdID = new VCSimulationDataIdentifier(vcSimID, 0); + + Cachetable cachetable = new Cachetable(10 * Cachetable.minute, 1000000L); + File primaryDir = new File(primaryDirStr); + DataSetControllerImpl dataSetControllerImpl = new DataSetControllerImpl(cachetable, primaryDir, null); + double[] allTimes = dataSetControllerImpl.getDataSetTimes(vcdID); + + // get dataset identifier for vcSimID + VCData vcData = dataSetControllerImpl.getVCData(vcdID); + OutputContext outputContext = new OutputContext(null); + DataIdentifier[] dataIdentifiers = vcData.getVarAndFunctionDataIdentifiers(outputContext); + DataIdentifier DexDataIdentifier = Arrays.stream(dataIdentifiers).filter(di -> di.getName().equals("Dex")).findFirst().get(); + SimDataBlock simDataBlock = vcData.getSimDataBlock(outputContext, DexDataIdentifier.getName(), allTimes[0]); + double[] Dex_dataBlockValues = simDataBlock.getData(); // X,Y,Z raster for time 0 for variable cAMP + System.out.println("Dex_dataBlockValues.length = " + Dex_dataBlockValues.length); + System.out.println("Dex_dataBlockValues[0] = " + Dex_dataBlockValues[0]); + + } +} \ No newline at end of file