Merge pull request #231 from EBISPOT/dev

dev

.gitlab-ci.yml

@@ -45,7 +45,7 @@ build_release:
    - tags
 
 
-deploy_staging:
+deploy_staging_app:
   image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/dtzar/helm-kubectl:2.13.1
   stage: deploy
   script:
@@ -55,15 +55,49 @@ deploy_staging:
     - RABBITPWD=$(kubectl --namespace rabbitmq get secret rabbitmq -o jsonpath="{.data.rabbitmq-password}" | base64 -d)
     - helm init --stable-repo-url https://charts.helm.sh/stable
     - helm delete --purge gwas-sumstats-service-dev || true
-    - helm install --name gwas-sumstats-service-dev --set k8Namespace=gwas-dev,image.repository=$CI_REGISTRY_IMAGE,image.tag=$CI_COMMIT_SHA,service.name=gwas-ss-service-dev,worker.name=gwas-ss-worker-dev,image.env.celeryUser=$RABBITUSER,image.env.celeryPwd=$RABBITPWD,image.env.celeryQueue1='prevalDev',image.env.celeryQueue2='postvalDev',image.env.gwasEndpointID=$GWAS_ENDPOINT_ID,image.env.globusSecret=$GLOBUS_SECRET,image.env.transferClientID=$TRANSFER_CLIENT_ID,image.env.clientID=$CLIENT_ID,image.env.ftpServer=$FTP_SERVER,image.env.ftpUser=$FTP_USERNAME,image.env.ftpPassword=$FTP_PASSWORD,image.env.mongoURI=$SANDBOX_MONGO_URI,image.env.mongoUser=$SANDBOX_MONGO_USER,image.env.mongoPassword=$SANDBOX_MONGO_PASSWORD,image.env.mongoDB=$SANDBOX_MONGO_DB,image.env.validatedPath=$VALIDATED_PATH_SANDBOX,image.env.gwasGlobusGroup=$GWAS_GLOBUS_GROUP,volume.log.ClaimName=gwas-dev-depo-logs,image.env.singularityCache=$SINGULARITY_CACHEDIR,image.env.storagePath=$STORAGE_PATH_SANDBOX,image.env.computeFarmLogin=$HH_LOGIN_NODE,image.env.computeFarmUser=$CLUSTER_USERNAME,image.env.computeFarmQueueLong=$COMPUTE_FARM_QUEUE_LONG,image.env.httpProxy=$HH_HTTP_PROXY,image.env.httpsProxy=$HH_HTTPS_PROXY,image.env.remoteHttpProxy=$PG_HTTP_PROXY,image.env.remoteHttpsProxy=$PG_HTTPS_PROXY,image.env.stagingPath=$STAGING_PATH_SANDBOX,image.env.swPath=$SW_PATH ./k8chart/ --wait
+    - helm install --name gwas-sumstats-service-dev --set k8Namespace=gwas-dev,image.repository=$CI_REGISTRY_IMAGE,image.tag=$CI_COMMIT_SHA,service.name=gwas-ss-service-dev,worker.name=gwas-ss-worker-dev,image.env.celeryUser=$RABBITUSER,image.env.celeryPwd=$RABBITPWD,image.env.celeryQueue1='prevalDev',image.env.celeryQueue2='postvalDev',image.env.gwasEndpointID=$GWAS_ENDPOINT_ID,image.env.globusSecret=$GLOBUS_SECRET,image.env.transferClientID=$TRANSFER_CLIENT_ID,image.env.clientID=$CLIENT_ID,image.env.ftpServer=$FTP_SERVER,image.env.ftpUser=$FTP_USERNAME,image.env.ftpPassword=$FTP_PASSWORD,image.env.mongoURI=$SANDBOX_MONGO_URI,image.env.mongoUser=$SANDBOX_MONGO_USER,image.env.mongoPassword=$SANDBOX_MONGO_PASSWORD,image.env.mongoDB=$SANDBOX_MONGO_DB,image.env.validatedPath=$VALIDATED_PATH_SANDBOX,image.env.gwasGlobusGroup=$GWAS_GLOBUS_GROUP,volume.log.ClaimName=gwas-dev-depo-logs,image.env.singularityCache=$SINGULARITY_CACHEDIR,image.env.storagePath=$STORAGE_PATH_SANDBOX,image.env.computeFarmLogin=$HH_LOGIN_NODE,image.env.computeFarmUser=$CLUSTER_USERNAME,image.env.computeFarmQueueLong=$COMPUTE_FARM_QUEUE_LONG,image.env.httpProxy=$HH_HTTP_PROXY,image.env.httpsProxy=$HH_HTTPS_PROXY,image.env.remoteHttpProxy=$PG_HTTP_PROXY,image.env.remoteHttpsProxy=$PG_HTTPS_PROXY,image.env.stagingPath=$STAGING_PATH_SANDBOX,image.env.swPath=$SW_PATH,worker.replicaCount=3,volume.data.nfsServer=$NFS_SERVER_PROD,volume.data.path=$NFS_PATH_PROD,image.env.uid=$UID,image.env.gid=$GID,image.env.depoAPIToken=$DEPO_API_AUTH_TOKEN,image.env.outputDataPath=$OUTPUT_PATH,image.env.depoPath=$DEPO_PATH ./k8chart/ --wait
   environment:
     name: sandbox
   only:
     - master
     - dev
 
+deploy_staging_workers:
+  image: alpine
+  stage: deploy
+  before_script:
+    - apk add openssh-client
+    - eval $(ssh-agent -s)
+    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
+    - mkdir -p ~/.ssh
+    - chmod 700 ~/.ssh
+  script:
+    - ssh -o StrictHostKeyChecking=no "$WORKER_USER"@"$WORKER_SERVER"
+      "bsub -K -cwd $WORKER_DIR_DEV -oo deploy.out -eo deploy.err -M12000 -R 'rusage[mem=12000]' ./START_CELERY_WORKERS.sh $CI_COMMIT_SHA"
+  tags:
+    - gwas
+  only:
+    - master
+    - dev
+
+deploy_prod_workers:
+  image: alpine
+  stage: deploy
+  before_script:
+    - apk add openssh-client
+    - eval $(ssh-agent -s)
+    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
+    - mkdir -p ~/.ssh
+    - chmod 700 ~/.ssh
+  script:
+    - ssh -o StrictHostKeyChecking=no "$WORKER_USER"@"$WORKER_SERVER"
+      "bsub -K -cwd $WORKER_DIR -oo deploy.out -eo deploy.err -M12000 -R 'rusage[mem=12000]' ./START_CELERY_WORKERS.sh $CI_COMMIT_SHA"
+  tags:
+    - gwas
+  only:
+    - tags
 
-deploy_fallback:
+deploy_fallback_app:
   image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/dtzar/helm-kubectl:2.13.1
   stage: deploy
   script:
@@ -81,7 +115,7 @@ deploy_fallback:
     - tags
 
 
-deploy_prod:
+deploy_prod_app:
   image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/dtzar/helm-kubectl:2.13.1
   stage: deploy
   script:
@@ -91,7 +125,7 @@ deploy_prod:
     - RABBITPWD=$(kubectl --namespace rabbitmq get secret rabbitmq -o jsonpath="{.data.rabbitmq-password}" | base64 -d)
     - helm init --stable-repo-url https://charts.helm.sh/stable
     - helm delete --purge gwas-sumstats-service || true
-    - helm install --name gwas-sumstats-service --set k8Namespace=gwas,image.repository=$CI_REGISTRY_IMAGE,image.tag=$CI_COMMIT_SHA,image.env.celeryUser=$RABBITUSER,image.env.celeryPwd=$RABBITPWD,image.env.storagePath=$STORAGE_PATH,image.env.computeFarmLogin=$HH_LOGIN_NODE,image.env.computeFarmUser=$CLUSTER_USERNAME,image.env.computeFarmQueueLong=$COMPUTE_FARM_QUEUE_LONG,image.env.httpProxy=$HH_HTTP_PROXY,image.env.httpsProxy=$HH_HTTPS_PROXY,image.env.gwasEndpointID=$GWAS_ENDPOINT_ID,image.env.globusSecret=$GLOBUS_SECRET,image.env.transferClientID=$TRANSFER_CLIENT_ID,image.env.clientID=$CLIENT_ID,image.env.ftpServer=$FTP_SERVER,image.env.ftpUser=$FTP_USERNAME,image.env.ftpPassword=$FTP_PASSWORD,image.env.mongoURI=$PROD_MONGO_URI,image.env.mongoUser=$PROD_MONGO_USER,image.env.mongoPassword=$PROD_MONGO_PASSWORD,image.env.mongoDB=$PROD_MONGO_DB,image.env.remoteHttpProxy=$PG_HTTP_PROXY,image.env.remoteHttpsProxy=$PG_HTTPS_PROXY,image.env.stagingPath=$STAGING_PATH,image.env.validatedPath=$VALIDATED_PATH,image.env.swPath=$SW_PATH,image.env.singularityCache=$SINGULARITY_CACHEDIR,image.env.gwasGlobusGroup=$GWAS_GLOBUS_GROUP,worker.replicaCount=3 ./k8chart/ --wait
+    - helm install --name gwas-sumstats-service --set k8Namespace=gwas,image.repository=$CI_REGISTRY_IMAGE,image.tag=$CI_COMMIT_SHA,image.env.celeryUser=$RABBITUSER,image.env.celeryPwd=$RABBITPWD,image.env.storagePath=$STORAGE_PATH,image.env.computeFarmLogin=$HH_LOGIN_NODE,image.env.computeFarmUser=$CLUSTER_USERNAME,image.env.computeFarmQueueLong=$COMPUTE_FARM_QUEUE_LONG,image.env.httpProxy=$HH_HTTP_PROX$STORAGE_PATH,image.env.computeFarmLogin=$HH_LOGIN_NODE,image.env.computeFarmUser=$CLUSTER_USERNAME,image.env.computeFarmQueueLong=$COMPUTE_FARM_QUEUE_LONG,image.env.httpProxy=$HH_Y,image.env.httpsProxy=$HH_HTTPS_PROXY,image.env.gwasEndpointID=$GWAS_ENDPOINT_ID,image.env.globusSecret=$GLOBUS_SECRET,image.env.transferClientID=$TRANSFER_CLIENT_ID,image.env.clientID=$CLIENT_ID,image.env.ftpServer=$FTP_SERVER,image.env.ftpUser=$FTP_USERNAME,image.env.ftpPassword=$FTP_PASSWORD,image.env.mongoURI=$PROD_MONGO_URI,image.env.mongoUser=$PROD_MONGO_USER,image.env.mongoPassword=$PROD_MONGO_PASSWORD,image.env.mongoDB=$PROD_MONGO_DB,image.env.remoteHttpProxy=$PG_HTTP_PROXY,image.env.remoteHttpsProxy=$PG_HTTPS_PROXY,image.env.stagingPath=$STAGING_PATH,image.env.validatedPath=$VALIDATED_PATH,image.env.swPath=$SW_PATH,image.env.singularityCache=$SINGULARITY_CACHEDIR,image.env.gwasGlobusGroup=$GWAS_GLOBUS_GROUP,worker.replicaCount=3 ./k8chart/ --wait
   environment:
     name: production
   when: manual

dockerfile

@@ -1,4 +1,6 @@
-FROM python:3.6-slim-buster
+FROM python:3.9-slim-buster
+
+RUN groupadd -r sumstats-service && useradd -r --create-home -g sumstats-service sumstats-service
 
 ENV INSTALL_PATH /sumstats_service
 RUN mkdir -p $INSTALL_PATH
@@ -8,6 +10,7 @@ COPY requirements.txt requirements.txt
 RUN apt-get update \
     && apt-get install -y --no-install-recommends gcc openssh-client python-dev libmagic-dev \
     && rm -rf /var/lib/apt/lists/* \
+    && pip install --upgrade pip \
     && pip install -r requirements.txt \
     && apt-get purge -y --auto-remove gcc python-dev
 # the --no-install-recommends helps limit some of the install so that you can be more explicit about what gets installed
@@ -20,6 +23,7 @@ RUN pip install -e .
 EXPOSE 8000
 
 RUN mkdir -p logs
+RUN chown -R sumstats-service:sumstats-service $INSTALL_PATH
 
 ENV CELERY_PROTOCOL "amqp"
 ENV CELERY_USER "guest"
@@ -30,6 +34,7 @@ ENV CELERY_QUEUE1 "preval"
 ENV CELERY_QUEUE2 "postval"
 ENV STORAGE_PATH "./data"
 ENV STAGING_PATH "./staging"
+ENV DEPO_PATH "./depo_data"
 ENV VALIDATED_PATH "./depo_ss_validated"
 ENV SW_PATH "./bin"
 ENV VALIDATE_WITH_SSH ""
@@ -56,4 +61,9 @@ ENV MONGO_DB ""
 ENV HTTP_PROXY ""
 ENV HTTPS_PROXY ""
 ENV no_proxy "localhost,.cluster.local"
+ENV DEPO_API_AUTH_TOKEN ""
+ENV OUTPUT_PATH "metadata/output"
+ENV SUMSTATS_FILE_TYPE ""
+ENV GWAS_DEPO_REST_API_URL ""
 
+USER sumstats-service

k8chart/templates/gwas-ss-service-deployment.yaml

@@ -30,6 +30,9 @@ spec:
         volumeMounts:
         - mountPath: {{.Values.image.logMountPath}}
           name: log
+        securityContext:
+          runAsUser: {{.Values.image.env.uid}}
+          runAsGroup: {{.Values.image.env.gid}}
         env:
         - name: CELERY_USER
           value: {{.Values.image.env.celeryUser}}

k8chart/values.yaml

@@ -10,8 +10,9 @@ image:
   tag: latest
   pullPolicy: Always
   containerPort: 8000
-  logMountPath: "/sumstats_service/logs"
-  sshMountPath: "/root/.ssh"
+  logMountPath: "/var/log/gunicorn"
+  sshMountPath: "/sumstats_service/.ssh"
+  dataMountPath: "/sumstats_service/data"
   env:
     celeryUser: "username"
     celeryPwd: "password"
@@ -40,13 +41,23 @@ image:
     httpsProxy: ""
     stagingPath: "./staging"
     validatedPath: "./validated"
+    depoPath: "./depo_data"
     swPath: "./bin"
     singularityCache: ""
+    depoAPIToken: ""
+    outputDataPath: "./output_data"
+    uid: 1000
+    gid: 1000
+    user: virtual_user
 
 
 volume:
   log:
     ClaimName: gwas-depo-logs
+  data:
+    ClaimName: gwas-sumstats-data
+    nfsServer: "<host name or IP>"
+    path: "/path/to/files/"
 
 secrets:
   sshName: ssh-keys
@@ -59,12 +70,12 @@ service:
   name: gwas-ss-service
   type: ClusterIP
   port: 8000
-  gunicornCmd: '["gunicorn", "-b", "0.0.0.0:8000", "sumstats_service.app:app", "-k=eventlet", "-t=1200", "-w=2", "--log-level=info","--access-logfile=logs/ss_access.log","--error-logfile=logs/ss_error.log"]'
+  gunicornCmd: '["gunicorn", "-b", "0.0.0.0:8000", "sumstats_service.app:app", "-k=eventlet", "-t=3600", "-w=2", "--log-config=log.conf"]'
 
 worker:
   name: gwas-ss-worker
-  celeryCmd: '["celery", "-A", "sumstats_service.app.celery", "worker", "--loglevel=info", "--queues=preval,postval"]'
-  celeryCmdPostOnly:  '["celery", "-A", "sumstats_service.app.celery", "worker", "--loglevel=info", "--queues=postval"]'
+  celeryCmd: '["celery", "-A", "sumstats_service.app.celery", "worker", "--loglevel=INFO", "--queues=preval,postval"]'
+  celeryCmdPostOnly:  '["celery", "-A", "sumstats_service.app.celery", "worker", "--loglevel=INFO", "--queues=postval"]'
   replicaCount: 1
 
 ingress:

log.conf

@@ -0,0 +1,48 @@
+[loggers]
+keys=root, gunicorn.error, gunicorn.access
+
+[handlers]
+keys=console, error_file, access_file
+
+[formatters]
+keys=generic, access
+
+[logger_root]
+level=INFO
+handlers=console
+
+[logger_gunicorn.error]
+level=INFO
+handlers=error_file
+propagate=1
+qualname=gunicorn.error
+
+[logger_gunicorn.access]
+level=INFO
+handlers=access_file
+propagate=0
+qualname=gunicorn.access
+
+[handler_console]
+class=StreamHandler
+formatter=generic
+args=(sys.stdout, )
+
+[handler_error_file]
+class=logging.handlers.TimedRotatingFileHandler
+formatter=generic
+args=('/var/log/gunicorn/sumstats-error.log', 'midnight', 1, 30, 'utf-8')
+
+[handler_access_file]
+class=logging.handlers.TimedRotatingFileHandler
+formatter=access
+args=('/var/log/gunicorn/sumstats-access.log', 'midnight', 1, 30, 'utf-8')
+
+[formatter_generic]
+format=%(asctime)s [%(process)d] [%(levelname)s] %(message)s
+datefmt=%Y-%m-%d %H:%M:%S
+class=logging.Formatter
+
+[formatter_access]
+format=%(message)s
+class=logging.Formatter