Merge pull request #238 from EBISPOT/dev

Globus V5 update
EBISPOT · May 10, 2023 · ad512fd · ad512fd
2 parents 4bed9d7 + 8c2f2ef
commit ad512fd
Show file tree

Hide file tree

Showing 8 changed files with 214 additions and 194 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -55,7 +55,7 @@ deploy_staging_app:
     - RABBITPWD=$(kubectl --namespace rabbitmq get secret rabbitmq -o jsonpath="{.data.rabbitmq-password}" | base64 -d)
     - helm init --stable-repo-url https://charts.helm.sh/stable
     - helm delete --purge gwas-sumstats-service-dev || true
-    - helm install --name gwas-sumstats-service-dev --set k8Namespace=gwas-dev,image.repository=$CI_REGISTRY_IMAGE,image.tag=$CI_COMMIT_SHA,service.name=gwas-ss-service-dev,worker.name=gwas-ss-worker-dev,image.env.celeryUser=$RABBITUSER,image.env.celeryPwd=$RABBITPWD,image.env.celeryQueue1='prevalDev',image.env.celeryQueue2='postvalDev',image.env.gwasEndpointID=$GWAS_ENDPOINT_ID,image.env.globusSecret=$GLOBUS_SECRET,image.env.transferClientID=$TRANSFER_CLIENT_ID,image.env.clientID=$CLIENT_ID,image.env.ftpServer=$FTP_SERVER,image.env.ftpUser=$FTP_USERNAME,image.env.ftpPassword=$FTP_PASSWORD,image.env.mongoURI=$SANDBOX_MONGO_URI,image.env.mongoUser=$SANDBOX_MONGO_USER,image.env.mongoPassword=$SANDBOX_MONGO_PASSWORD,image.env.mongoDB=$SANDBOX_MONGO_DB,image.env.validatedPath=$VALIDATED_PATH_SANDBOX,image.env.gwasGlobusGroup=$GWAS_GLOBUS_GROUP,volume.log.ClaimName=gwas-dev-depo-logs,image.env.singularityCache=$SINGULARITY_CACHEDIR,image.env.storagePath=$STORAGE_PATH_SANDBOX,image.env.computeFarmLogin=$HH_LOGIN_NODE,image.env.computeFarmUser=$CLUSTER_USERNAME,image.env.computeFarmQueueLong=$COMPUTE_FARM_QUEUE_LONG,image.env.httpProxy=$HH_HTTP_PROXY,image.env.httpsProxy=$HH_HTTPS_PROXY,image.env.remoteHttpProxy=$PG_HTTP_PROXY,image.env.remoteHttpsProxy=$PG_HTTPS_PROXY,image.env.stagingPath=$STAGING_PATH_SANDBOX,image.env.swPath=$SW_PATH,worker.replicaCount=3,volume.data.nfsServer=$NFS_SERVER_PROD,volume.data.path=$NFS_PATH_PROD,image.env.uid=$UID,image.env.gid=$GID,image.env.depoAPIToken=$DEPO_API_AUTH_TOKEN,image.env.outputDataPath=$OUTPUT_PATH,image.env.depoPath=$DEPO_PATH ./k8chart/ --wait
+    - helm install --name gwas-sumstats-service-dev --set k8Namespace=gwas-dev,image.repository=$CI_REGISTRY_IMAGE,image.tag=$CI_COMMIT_SHA,service.name=gwas-ss-service-dev,worker.name=gwas-ss-worker-dev,image.env.celeryUser=$RABBITUSER,image.env.celeryPwd=$RABBITPWD,image.env.celeryQueue1='prevalDev',image.env.celeryQueue2='postvalDev',image.env.gwasEndpointID=$GWAS_ENDPOINT_ID_DEV,image.env.globusSecret=$CLIENT_SECRET,image.env.transferClientID=$TRANSFER_CLIENT_ID_DEV,image.env.clientID=$CLIENT_ID_DEV,image.env.gwasGlobusGroup=$GWAS_GLOBUS_GROUP,image.env.globusHostname=$GLOBUS_HOSTNAME,image.env.mappedCollectionId=$MAPPED_COLLECTION_ID,image.env.storageGatewayId=$STORAGE_GATEWAY_ID,image.env.ftpServer=$FTP_SERVER,image.env.ftpUser=$FTP_USERNAME,image.env.ftpPassword=$FTP_PASSWORD,image.env.mongoURI=$SANDBOX_MONGO_URI,image.env.mongoUser=$SANDBOX_MONGO_USER,image.env.mongoPassword=$SANDBOX_MONGO_PASSWORD,image.env.mongoDB=$SANDBOX_MONGO_DB,image.env.validatedPath=$VALIDATED_PATH_SANDBOX,volume.log.ClaimName=gwas-dev-depo-logs,image.env.singularityCache=$SINGULARITY_CACHEDIR,image.env.storagePath=$STORAGE_PATH_SANDBOX,image.env.computeFarmLogin=$HH_LOGIN_NODE,image.env.computeFarmUser=$CLUSTER_USERNAME,image.env.computeFarmQueueLong=$COMPUTE_FARM_QUEUE_LONG,image.env.httpProxy=$HH_HTTP_PROXY,image.env.httpsProxy=$HH_HTTPS_PROXY,image.env.remoteHttpProxy=$PG_HTTP_PROXY,image.env.remoteHttpsProxy=$PG_HTTPS_PROXY,image.env.stagingPath=$STAGING_PATH_SANDBOX,image.env.swPath=$SW_PATH,replicaCount=1,volume.data.nfsServer=$NFS_SERVER_PROD,volume.data.path=$NFS_PATH_PROD,image.env.uid=$UID,image.env.gid=$GID,image.env.depoAPIToken=$DEPO_API_AUTH_TOKEN,image.env.outputDataPath=$OUTPUT_PATH,image.env.depoPath=$DEPO_PATH ./k8chart/ --wait
   environment:
     name: sandbox
   only:
@@ -91,7 +91,7 @@ deploy_prod_workers:
     - chmod 700 ~/.ssh
   script:
     - ssh -o StrictHostKeyChecking=no "$WORKER_USER"@"$WORKER_SERVER"
-      "bsub -K -cwd $WORKER_DIR -oo deploy.out -eo deploy.err -M12000 -R 'rusage[mem=12000]' ./START_CELERY_WORKERS.sh $CI_COMMIT_SHA"
+      "bsub -K -cwd $WORKER_DIR -oo deploy.out -eo deploy.err -M12000 -R 'rusage[mem=12000]' ./START_CELERY_WORKERS.sh"
   tags:
     - gwas
   only:
@@ -107,7 +107,7 @@ deploy_fallback_app:
     - RABBITPWD=$(kubectl --namespace rabbitmq get secret rabbitmq -o jsonpath="{.data.rabbitmq-password}" | base64 -d)
     - helm init --stable-repo-url https://charts.helm.sh/stable
     - helm delete --purge gwas-sumstats-service || true
-    - helm install --name gwas-sumstats-service --set k8Namespace=gwas,image.repository=$CI_REGISTRY_IMAGE,image.tag=$CI_COMMIT_SHA,image.env.celeryUser=$RABBITUSER,image.env.celeryPwd=$RABBITPWD,image.env.storagePath=$STORAGE_PATH,image.env.computeFarmLogin=$HX_LOGIN_NODE,image.env.computeFarmUser=$CLUSTER_USERNAME,image.env.computeFarmQueueLong=$COMPUTE_FARM_QUEUE_LONG,image.env.httpProxy=$HX_HTTP_PROXY,image.env.httpsProxy=$HX_HTTPS_PROXY,image.env.gwasEndpointID=$GWAS_ENDPOINT_ID,image.env.globusSecret=$GLOBUS_SECRET,image.env.transferClientID=$TRANSFER_CLIENT_ID,image.env.clientID=$CLIENT_ID,image.env.ftpServer=$FTP_SERVER,image.env.ftpUser=$FTP_USERNAME,image.env.ftpPassword=$FTP_PASSWORD,image.env.mongoURI=$PROD_MONGO_URI,image.env.mongoUser=$PROD_MONGO_USER,image.env.mongoPassword=$PROD_MONGO_PASSWORD,image.env.mongoDB=$PROD_MONGO_DB,image.env.remoteHttpProxy=$HX_HTTP_PROXY,image.env.remoteHttpsProxy=$HX_HTTPS_PROXY,image.env.stagingPath=$STAGING_PATH,image.env.validatedPath=$VALIDATED_PATH,image.env.swPath=$SW_PATH,image.env.singularityCache=$SINGULARITY_CACHEDIR,image.env.gwasGlobusGroup=$GWAS_GLOBUS_GROUP,worker.replicaCount=3  ./k8chart/ --wait
+    - helm install --name gwas-sumstats-service --set k8Namespace=gwas,image.repository=$CI_REGISTRY_IMAGE,image.tag=$CI_COMMIT_SHA,image.env.celeryUser=$RABBITUSER,image.env.celeryPwd=$RABBITPWD,image.env.storagePath=$STORAGE_PATH,image.env.computeFarmLogin=$HX_LOGIN_NODE,image.env.computeFarmUser=$CLUSTER_USERNAME,image.env.computeFarmQueueLong=$COMPUTE_FARM_QUEUE_LONG,image.env.httpProxy=$HX_HTTP_PROXY,image.env.httpsProxy=$HX_HTTPS_PROXY,image.env.gwasEndpointID=$GWAS_ENDPOINT_ID,image.env.globusSecret=$CLIENT_SECRET,image.env.transferClientID=$TRANSFER_CLIENT_ID,image.env.clientID=$CLIENT_ID,image.env.ftpServer=$FTP_SERVER,image.env.ftpUser=$FTP_USERNAME,image.env.ftpPassword=$FTP_PASSWORD,image.env.mongoURI=$PROD_MONGO_URI,image.env.mongoUser=$PROD_MONGO_USER,image.env.mongoPassword=$PROD_MONGO_PASSWORD,image.env.mongoDB=$PROD_MONGO_DB,image.env.remoteHttpProxy=$HX_HTTP_PROXY,image.env.remoteHttpsProxy=$HX_HTTPS_PROXY,image.env.stagingPath=$STAGING_PATH,image.env.validatedPath=$VALIDATED_PATH,image.env.swPath=$SW_PATH,image.env.singularityCache=$SINGULARITY_CACHEDIR,image.env.gwasGlobusGroup=$GWAS_GLOBUS_GROUP,replicaCount=3,image.env.globusHostname=$GLOBUS_HOSTNAME,image.env.mappedCollectionId=$MAPPED_COLLECTION_ID,image.env.storageGatewayId=$STORAGE_GATEWAY_ID  ./k8chart/ --wait
   environment:
     name: fallback
   when: manual
@@ -125,7 +125,7 @@ deploy_prod_app:
     - RABBITPWD=$(kubectl --namespace rabbitmq get secret rabbitmq -o jsonpath="{.data.rabbitmq-password}" | base64 -d)
     - helm init --stable-repo-url https://charts.helm.sh/stable
     - helm delete --purge gwas-sumstats-service || true
-    - helm install --name gwas-sumstats-service --set k8Namespace=gwas,image.repository=$CI_REGISTRY_IMAGE,image.tag=$CI_COMMIT_SHA,image.env.celeryUser=$RABBITUSER,image.env.celeryPwd=$RABBITPWD,image.env.storagePath=$STORAGE_PATH,image.env.computeFarmLogin=$HH_LOGIN_NODE,image.env.computeFarmUser=$CLUSTER_USERNAME,image.env.computeFarmQueueLong=$COMPUTE_FARM_QUEUE_LONG,image.env.httpProxy=$HH_HTTP_PROX$STORAGE_PATH,image.env.computeFarmLogin=$HH_LOGIN_NODE,image.env.computeFarmUser=$CLUSTER_USERNAME,image.env.computeFarmQueueLong=$COMPUTE_FARM_QUEUE_LONG,image.env.httpProxy=$HH_Y,image.env.httpsProxy=$HH_HTTPS_PROXY,image.env.gwasEndpointID=$GWAS_ENDPOINT_ID,image.env.globusSecret=$GLOBUS_SECRET,image.env.transferClientID=$TRANSFER_CLIENT_ID,image.env.clientID=$CLIENT_ID,image.env.ftpServer=$FTP_SERVER,image.env.ftpUser=$FTP_USERNAME,image.env.ftpPassword=$FTP_PASSWORD,image.env.mongoURI=$PROD_MONGO_URI,image.env.mongoUser=$PROD_MONGO_USER,image.env.mongoPassword=$PROD_MONGO_PASSWORD,image.env.mongoDB=$PROD_MONGO_DB,image.env.remoteHttpProxy=$PG_HTTP_PROXY,image.env.remoteHttpsProxy=$PG_HTTPS_PROXY,image.env.stagingPath=$STAGING_PATH,image.env.validatedPath=$VALIDATED_PATH,image.env.swPath=$SW_PATH,image.env.singularityCache=$SINGULARITY_CACHEDIR,image.env.gwasGlobusGroup=$GWAS_GLOBUS_GROUP,worker.replicaCount=3 ./k8chart/ --wait
+    - helm install --name gwas-sumstats-service --set k8Namespace=gwas,image.repository=$CI_REGISTRY_IMAGE,image.tag=$CI_COMMIT_SHA,image.env.celeryUser=$RABBITUSER,image.env.celeryPwd=$RABBITPWD,image.env.storagePath=$STORAGE_PATH,image.env.computeFarmLogin=$HH_LOGIN_NODE,image.env.computeFarmUser=$CLUSTER_USERNAME,image.env.computeFarmQueueLong=$COMPUTE_FARM_QUEUE_LONG,image.env.httpProxy=$HH_HTTP_PROX$STORAGE_PATH,image.env.computeFarmLogin=$HH_LOGIN_NODE,image.env.computeFarmUser=$CLUSTER_USERNAME,image.env.computeFarmQueueLong=$COMPUTE_FARM_QUEUE_LONG,image.env.httpProxy=$HH_Y,image.env.httpsProxy=$HH_HTTPS_PROXY,image.env.gwasEndpointID=$GWAS_ENDPOINT_ID,image.env.globusSecret=$CLIENT_SECRET,image.env.transferClientID=$TRANSFER_CLIENT_ID,image.env.clientID=$CLIENT_ID,image.env.ftpServer=$FTP_SERVER,image.env.ftpUser=$FTP_USERNAME,image.env.ftpPassword=$FTP_PASSWORD,image.env.mongoURI=$PROD_MONGO_URI,image.env.mongoUser=$PROD_MONGO_USER,image.env.mongoPassword=$PROD_MONGO_PASSWORD,image.env.mongoDB=$PROD_MONGO_DB,image.env.remoteHttpProxy=$PG_HTTP_PROXY,image.env.remoteHttpsProxy=$PG_HTTPS_PROXY,image.env.stagingPath=$STAGING_PATH,image.env.validatedPath=$VALIDATED_PATH,image.env.swPath=$SW_PATH,image.env.singularityCache=$SINGULARITY_CACHEDIR,image.env.gwasGlobusGroup=$GWAS_GLOBUS_GROUP,replicaCount=3,image.env.globusHostname=$GLOBUS_HOSTNAME,image.env.mappedCollectionId=$MAPPED_COLLECTION_ID,image.env.storageGatewayId=$STORAGE_GATEWAY_ID ./k8chart/ --wait
   environment:
     name: production
   when: manual

diff --git a/dockerfile b/dockerfile
@@ -47,10 +47,13 @@ ENV SINGULARITY_CACHEDIR ""
 ENV REMOTE_HTTP_PROXY ""
 ENV REMOTE_HTTPS_PROXY ""
 ENV GWAS_ENDPOINT_ID ""
-ENV GLOBUS_SECRET ""
+ENV CLIENT_SECRET ""
 ENV TRANSFER_CLIENT_ID ""
 ENV GWAS_GLOBUS_GROUP ""
 ENV CLIENT_ID ""
+ENV GLOBUS_HOSTNAME ""
+ENV MAPPED_COLLECTION_ID ""
+ENV STORAGE_GATEWAY_ID ""
 ENV FTP_SERVER ""
 ENV FTP_USERNAME ""
 ENV FTP_PASSWORD ""

diff --git a/k8chart/templates/gwas-ss-service-deployment.yaml b/k8chart/templates/gwas-ss-service-deployment.yaml
@@ -44,14 +44,20 @@ spec:
           value: {{.Values.image.env.celeryQueue2}}
         - name: GWAS_ENDPOINT_ID
           value: {{.Values.image.env.gwasEndpointID}}
-        - name: GLOBUS_SECRET
+        - name: CLIENT_SECRET
           value: {{.Values.image.env.globusSecret}}
         - name: GWAS_GLOBUS_GROUP
           value: {{.Values.image.env.gwasGlobusGroup}}
         - name: TRANSFER_CLIENT_ID
           value: {{.Values.image.env.transferClientID}}
         - name: CLIENT_ID
           value: {{.Values.image.env.clientID}}
+        - name: GLOBUS_HOSTNAME
+          value: {{.Values.image.env.globusHostname}}
+        - name: MAPPED_COLLECTION_ID
+          value: {{.Values.image.env.mappedCollectionId}}
+        - name: STORAGE_GATEWAY_ID
+          value: {{.Values.image.env.storageGatewayId}}
         - name: MONGO_URI
           value: {{.Values.image.env.mongoURI}}
         - name: MONGO_USER

diff --git a/k8chart/values.yaml b/k8chart/values.yaml
@@ -30,6 +30,9 @@ image:
     transferClientID: ""
     gwasGlobusGroup: ""
     clientID: ""
+    globusHostname: ""
+    mappedCollectionId: ""
+    storageGatewayId: ""
     ftpServer: ""
     ftpUser: ""
     ftpPassword: ""

diff --git a/requirements.txt b/requirements.txt
@@ -1,6 +1,9 @@
 amqp==2.5.0
+appnope==0.1.3
+asttokens==2.2.1
 atomicwrites==1.3.0
 attrs==22.2.0
+backcall==0.2.0
 bcrypt==3.1.7
 billiard==3.6.0.0
 black==23.1.0
@@ -15,37 +18,43 @@ click==8.1.3
 colorama==0.4.6
 commonmark==0.9.1
 coverage==4.5.3
-cryptography==2.8
+cryptography==36.0.2
 daiquiri==1.5.0
 decorator==5.1.1
 distlib==0.3.6
 dnspython==1.16.0
 et-xmlfile==1.1.0
 eventlet==0.25.1
 exceptiongroup==1.1.0
+executing==1.2.0
 extras==1.0.0
 filelock==3.9.0
 fixtures==3.0.0
 Flask==2.2.3
 frictionless==5.6.3
 gdown==3.8.3
-globus-sdk==1.8.0
+globus-cli==3.12.0
+globus-sdk==3.17.0
 greenlet==2.0.2
 gunicorn==19.9.0
-gwas-sumstats-tools==1.0.0
+gwas-sumstats-tools==0.1.3a0
 humanize==4.6.0
 hypothesis==6.68.2
 idna==3.4
 importlib-metadata==6.0.0
 iniconfig==2.0.0
+ipython==8.12.0
 isodate==0.6.1
 itsdangerous==2.1.2
+jedi==0.18.2
 Jinja2==3.1.2
+jmespath==1.0.1
 jsonschema==4.17.3
 kombu==4.6.3
 linecache2==1.0.0
 marko==1.3.0
 MarkupSafe==2.1.2
+matplotlib-inline==0.1.6
 mirakuru==2.0.1
 monotonic==1.5
 more-itertools==7.1.0
@@ -55,29 +64,35 @@ openpyxl==3.0.10
 packaging==23.0
 pamqp==2.3.0
 pandas==1.5.3
-pandas-schema==0.3.4
 pandera==0.13.4
 paramiko==2.6.0
+parso==0.8.3
 pathspec==0.11.0
 pbr==5.4.1
 petl==1.7.12
+pexpect==4.8.0
+pickleshare==0.7.5
 pifpaf==2.2.2
 platformdirs==3.0.0
 pluggy==1.0.0
 port-for==0.4
+prompt-toolkit==3.0.38
 psutil==5.6.6
+ptyprocess==0.7.0
+pure-eval==0.2.2
 py==1.8.0
 pycparser==2.19
 pydantic==1.10.5
 Pygments==2.14.0
-PyJWT==1.7.1
+PyJWT==2.6.0
 pymongo==4.3.3
 PyNaCl==1.3.0
 pyparsing==2.4.0
 pyproject_api==1.5.0
 pyrsistent==0.19.3
 pytest==7.2.1
 pytest-cov==2.7.1
+pytest-mock==3.10.0
 python-dateutil==2.8.2
 python-magic==0.4.15
 python-mimeparse==1.6.0
@@ -95,7 +110,7 @@ simpleeval==0.9.13
 simplejson==3.16.0
 six==1.16.0
 sortedcontainers==2.4.0
-ss-validate==1.0.0.dev3
+stack-data==0.6.2
 stringcase==1.2.0
 tabulate==0.9.0
 testtools==2.3.0
@@ -105,6 +120,7 @@ tomli==2.0.1
 tox==4.4.6
 tqdm==4.64.1
 traceback2==1.4.0
+traitlets==5.9.0
 typer==0.7.0
 typing-inspect==0.8.0
 typing_extensions==4.5.0
@@ -119,4 +135,4 @@ wget==3.2
 wincertstore==0.2
 wrapt==1.14.1
 xattr==0.9.6
-zipp==0.5.2
+zipp==0.5.2
diff --git a/sumstats_service/config.py b/sumstats_service/config.py
@@ -56,12 +56,16 @@ def _env_variable_else(env_var_name, default):
 SCOPES = ('openid email profile '
           'urn:globus:auth:scope:transfer.api.globus.org:all')
 GWAS_ENDPOINT_ID = _env_variable_else('GWAS_ENDPOINT_ID', None)
-GLOBUS_SECRET = _env_variable_else('GLOBUS_SECRET', None)
+CLIENT_SECRET = _env_variable_else('CLIENT_SECRET', None)
 CLIENT_ID = _env_variable_else('CLIENT_ID', None)
 TRANSFER_CLIENT_ID = _env_variable_else('TRANSFER_CLIENT_ID', None)
 GWAS_GLOBUS_GROUP = _env_variable_else('GWAS_GLOBUS_GROUP', None)
+GLOBUS_HOSTNAME = _env_variable_else('GLOBUS_HOSTNAME', None)
 DEPO_API_AUTH_TOKEN = _env_variable_else('DEPO_API_AUTH_TOKEN', None)
 OUTPUT_PATH = _env_variable_else('OUTPUT_PATH', 'metadata/output')
+MAPPED_COLLECTION_ID = _env_variable_else('MAPPED_COLLECTION_ID', None)
+STORAGE_GATEWAY_ID = _env_variable_else('STORAGE_GATEWAY_ID', None)
+GWAS_IDENTITY = _env_variable_else('GWAS_IDENTITY', '66dab3b3-b880-4017-b496-9643da909b89')
 
 # --- SQLite schema --- # 
 

diff --git a/sumstats_service/resources/file_handler.py b/sumstats_service/resources/file_handler.py
@@ -10,7 +10,6 @@
 import csv
 import io
 import logging
-import ss_validate.validator as val
 from gwas_sumstats_tools.validate import Validator
 import pathlib
 import sumstats_service.resources.globus as globus