Merge pull request #223 from bento-platform/releases/v15.2

Release v15.2

.gitignore

@@ -47,6 +47,10 @@ lib/public/*
 !lib/public/.gitkeep
 !lib/public/docker-compose.public.yaml
 
+# gohan_elasticsearch
+lib/gohan/es_jvm_options/*
+!lib/gohan/es_jvm_options/.gitkeep
+
 # temp test stuff
 etc/tests/*/__pycache__
 etc/tests/*/*/__pycache__

README.md

@@ -44,6 +44,7 @@ that make up the Bento platform.
 
 ### Migration documents
 
+* [v15.1 to v15.2](./docs/migrating_to_15_2.md)
 * [v15 to v15.1](./docs/migrating_to_15_1.md)
 * [v14 to v15](./docs/migrating_to_15.md)
   * [Converting Phenopackets from V1 to V2 using `bentoctl`](./docs/phenopackets_v1_to_v2.md) 

docker-compose.yaml

@@ -39,6 +39,7 @@ services:
 
       - BENTOV2_AUTH_CONTAINER_NAME
       - BENTOV2_AUTH_INTERNAL_PORT
+      - BENTOV2_AUTH_INTERNAL_PORT_PLAIN_HTTP
 
       - BENTO_AUTHZ_CONTAINER_NAME
       - BENTO_AUTHZ_INTERNAL_PORT
@@ -76,64 +77,22 @@ services:
       - BENTO_CBIOPORTAL_CONTAINER_NAME
       - BENTO_CBIOPORTAL_INTERNAL_PORT
     networks:
-      aggregation-net:
-        aliases:
-          - ${BENTOV2_DOMAIN}
-          - ${BENTOV2_PORTAL_DOMAIN}
-      auth-net:
-      authz-net:
-        aliases:
-          - ${BENTOV2_DOMAIN}
-          - ${BENTOV2_PORTAL_DOMAIN}
-      beacon-net:
-        aliases:
-          - ${BENTOV2_DOMAIN}
-          - ${BENTOV2_PORTAL_DOMAIN}
-      cbioportal-net:  # No aliases needed here; container for sure does not contact the gateway
-      drop-box-net:
-        aliases:
-          - ${BENTOV2_DOMAIN}
-          - ${BENTOV2_PORTAL_DOMAIN}
-      drs-net:
-        aliases:
-          - ${BENTOV2_DOMAIN}
-          - ${BENTOV2_PORTAL_DOMAIN}
-      event-relay-net:
-        aliases:
-          - ${BENTOV2_DOMAIN}
-          - ${BENTOV2_PORTAL_DOMAIN}
-      gohan-api-net:
-        aliases:
-          - ${BENTOV2_DOMAIN}
-          - ${BENTOV2_PORTAL_DOMAIN}
-      katsu-net:
-        aliases:
-          - ${BENTOV2_DOMAIN}
-          - ${BENTOV2_PORTAL_DOMAIN}
-      notification-net:
-        aliases:
-          - ${BENTOV2_DOMAIN}
-          - ${BENTOV2_PORTAL_DOMAIN}
-      public-net:
-        aliases:
-          - ${BENTOV2_DOMAIN}
-          - ${BENTOV2_PORTAL_DOMAIN}
-      reference-net:
-        aliases:
-          - ${BENTOV2_DOMAIN}
-          - ${BENTOV2_PORTAL_DOMAIN}
-      service-registry-net:
-        aliases:
-          - ${BENTOV2_DOMAIN}
-          - ${BENTOV2_PORTAL_DOMAIN}
-      web-net:
-        aliases:
-          - ${BENTOV2_DOMAIN}
-          - ${BENTOV2_PORTAL_DOMAIN}
-      wes-net:
-        aliases:
-          - ${BENTOV2_DOMAIN}
-          - ${BENTOV2_PORTAL_DOMAIN}
+      - aggregation-net
+      - auth-net
+      - authz-net
+      - beacon-net
+      - cbioportal-net
+      - drop-box-net
+      - drs-net
+      - event-relay-net
+      - gohan-api-net
+      - katsu-net
+      - notification-net
+      - public-net
+      - reference-net
+      - service-registry-net
+      - web-net
+      - wes-net
     ports:
       - "80:80"
       - "443:443"

docs/installation.md

@@ -126,7 +126,8 @@ BENTOV2_AUTH_REALM=bentov2
 BENTOV2_AUTH_WELLKNOWN_PATH=/auth/realms/${BENTOV2_AUTH_REALM}/.well-known/openid-configuration
 ```
 
-### `bento_public` configuration
+
+### Bento Public configuration
 
 Then, copy the `bento_public` configuration file to its correct location for use by Katsu, 
 Bento's clinical/phenotypic metadata service:
@@ -138,6 +139,45 @@ cp ./etc/katsu.config.example.json ./lib/katsu/config.json
 ```
 
 
+### Gohan configuration
+
+#### Production
+
+When deploying a Bento node that needs to serve variants data, it is recommended to allot larger portions of your 
+compute resources to both `bentov2-gohan-api` and `bentov2-gohan-elasticsearch`.
+Since variants are stored in memory in Elasticsearch, set the `BENTOV2_GOHAN_ES_MEM_LIM` variable to an 
+appropriate value.
+
+
+Alloting additional CPUs with `BENTOV2_GOHAN_API_CPUS` and `BENTOV2_GOHAN_ES_CPUS` will result in shorter ingestion
+times. Consider this option if you need to ingest a large number of VCFs.
+
+Go through the official Elasticsearch 
+[checklist](https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html#docker-prod-prerequisites) 
+for production deployments on Docker.
+
+The following configurations **MUST** be applied to the host machine deploying the Elasticsearch container:
+* [Set `vm.max_map_count`](https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html#_set_vm_max_map_count_to_at_least_262144)
+* [Disable swapping](https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-configuration-memory.html#swappiness)
+
+
+#### Elasticsearch JVM options
+
+In **production**, using the default JVM options provided by Elasticsearch is **recommended**.
+
+In **development**, modifying the JVM heap size could be needed if Elasticsearch's memory limit 
+is low and you are trying to ingest VCFs.
+The heap size can be modified by providing a config file to the `bentov2_gohan-elasticsearch` container:
+
+```bash
+cp ./etc/default.gohan.es.jvm.options ./lib/gohan/es_jvm_options/jvm.options
+```
+
+Set `Xms` and `Xmx` to no more than 50% of the `BENTOV2_GOHAN_ES_MEM_LIM` value.
+For more details, check the official Elasticsearch doc on 
+[heap size](https://www.elastic.co/guide/en/elasticsearch/reference/current/advanced-configuration.html#set-jvm-heap-size).
+
+
 ## 3. *Development only:* create self-signed TLS certificates 
 
 First, set up your local Bento and Keycloak hostnames (something like `bentov2.local`, `portal.bentov2.local`, and 

docs/migrating_to_15_2.md

@@ -0,0 +1,18 @@
+# Migrating to Bento v15.2
+
+Migrating to version 15.2 from version 15.1 should be straightforward.
+
+
+## 1. Update Gohan JVM configuration, if necessary
+
+See the [relevant section in the installation guide](installation.md#gohan-configuration) for more information.
+
+
+## 2. Update services and restart
+
+Run the following commands to pull the latest service images and restart services as needed:
+
+```bash
+./bentoctl.bash pull
+./bentoctl.bash restart
+```

etc/bento.env

@@ -31,7 +31,7 @@ BENTOV2_GATEWAY_INTERNAL_CERTS_DIR=/usr/local/openresty/nginx/certs
 
 # Gateway
 BENTOV2_GATEWAY_IMAGE=ghcr.io/bento-platform/bento_gateway
-BENTOV2_GATEWAY_VERSION=0.10.0
+BENTOV2_GATEWAY_VERSION=0.10.2
 BENTOV2_GATEWAY_VERSION_DEV=${BENTOV2_GATEWAY_VERSION}-dev
 BENTOV2_GATEWAY_CONTAINER_NAME=${BENTOV2_PREFIX}-gateway
 
@@ -44,11 +44,12 @@ BENTOV2_GATEWAY_CPUS=2
 
 # - Keycloak IdP - 'auth'
 BENTOV2_AUTH_IMAGE=ghcr.io/bento-platform/bento_keycloak_dist
-BENTOV2_AUTH_VERSION=2023.10.31
+BENTOV2_AUTH_VERSION=2024.02.16
 BENTOV2_AUTH_CONTAINER_NAME=${BENTOV2_PREFIX}-auth
 BENTO_AUTH_NETWORK=${BENTOV2_PREFIX}-auth-net
 BENTOV2_AUTH_SERVICE_HOST=0.0.0.0
 BENTOV2_AUTH_INTERNAL_PORT=8443
+BENTOV2_AUTH_INTERNAL_PORT_PLAIN_HTTP=8080
 BENTOV2_AUTH_VOL_DIR=${BENTOV2_ROOT_DATA_DIR}/auth/data
 
 BENTOV2_AUTH_WELLKNOWN_PATH=/realms/${BENTOV2_AUTH_REALM}/.well-known/openid-configuration
@@ -76,7 +77,7 @@ BENTO_AUTH_DB_NETWORK="${BENTOV2_PREFIX}-auth-db-net"
 
 # - Authz service
 BENTO_AUTHZ_IMAGE=ghcr.io/bento-platform/bento_authorization_service
-BENTO_AUTHZ_VERSION=0.7.0
+BENTO_AUTHZ_VERSION=0.8.0
 BENTO_AUTHZ_VERSION_DEV=${BENTO_AUTHZ_VERSION}-dev
 BENTO_AUTHZ_CONTAINER_NAME=${BENTOV2_PREFIX}-authz
 BENTO_AUTHZ_NETWORK=${BENTOV2_PREFIX}-authz-net
@@ -122,7 +123,7 @@ BENTOV2_DROP_BOX_CPUS=3
 
 # Service Registry
 BENTOV2_SERVICE_REGISTRY_IMAGE=ghcr.io/bento-platform/bento_service_registry
-BENTOV2_SERVICE_REGISTRY_VERSION=1.3.5
+BENTOV2_SERVICE_REGISTRY_VERSION=1.3.6
 BENTOV2_SERVICE_REGISTRY_VERSION_DEV=${BENTOV2_SERVICE_REGISTRY_VERSION}-dev
 BENTOV2_SERVICE_REGISTRY_CONTAINER_NAME=${BENTOV2_PREFIX}-service-registry
 BENTO_SERVICE_REGISTRY_NETWORK=${BENTOV2_PREFIX}-service-registry-net
@@ -134,7 +135,7 @@ BENTOV2_SERVICE_REGISTRY_CPUS=1
 
 # Notification
 BENTOV2_NOTIFICATION_IMAGE=ghcr.io/bento-platform/bento_notification_service
-BENTOV2_NOTIFICATION_VERSION=3.1.0
+BENTOV2_NOTIFICATION_VERSION=3.1.1
 BENTOV2_NOTIFICATION_VERSION_DEV=${BENTOV2_NOTIFICATION_VERSION}-dev
 BENTOV2_NOTIFICATION_CONTAINER_NAME=${BENTOV2_PREFIX}-notification
 BENTO_NOTIFICATION_NETWORK=${BENTOV2_PREFIX}-notification-net
@@ -164,7 +165,7 @@ BENTOV2_AGGREGATION_CPUS=2
 
 # Event-Relay
 BENTOV2_EVENT_RELAY_IMAGE=ghcr.io/bento-platform/bento_event_relay
-BENTOV2_EVENT_RELAY_VERSION=3.1.0
+BENTOV2_EVENT_RELAY_VERSION=3.1.1
 BENTOV2_EVENT_RELAY_VERSION_DEV=${BENTOV2_EVENT_RELAY_VERSION}-dev
 BENTOV2_EVENT_RELAY_CONTAINER_NAME=${BENTOV2_PREFIX}-event-relay
 BENTO_EVENT_RELAY_NETWORK=${BENTOV2_PREFIX}-event-relay-net
@@ -226,7 +227,7 @@ BENTOV2_WES_WORKFLOW_TIMEOUT=172800
 
 # DRS
 BENTOV2_DRS_IMAGE=ghcr.io/bento-platform/bento_drs
-BENTOV2_DRS_VERSION=0.15.2
+BENTOV2_DRS_VERSION=0.15.3
 BENTOV2_DRS_VERSION_DEV=${BENTOV2_DRS_VERSION}-dev
 BENTOV2_DRS_CONTAINER_NAME=${BENTOV2_PREFIX}-drs
 BENTO_DRS_NETWORK=${BENTOV2_PREFIX}-drs-net
@@ -295,7 +296,7 @@ CHORD_METADATA_SUB_PATH=/api/metadata
 
 # Redis
 BENTOV2_REDIS_BASE_IMAGE=redis
-BENTOV2_REDIS_BASE_IMAGE_VERSION=7.0.14-alpine
+BENTOV2_REDIS_BASE_IMAGE_VERSION=7.0.15-alpine
 BENTOV2_REDIS_CONTAINER_NAME=${BENTOV2_PREFIX}-redis
 BENTO_REDIS_NETWORK=${BENTOV2_PREFIX}-redis-net
 BENTOV2_REDIS_INTERNAL_PORT=6379
@@ -356,8 +357,8 @@ BENTOV2_GOHAN_ES_MEM_LIM=4G
 BENTOV2_GOHAN_ES_CPUS=4
 
 # allocate a little bit less memory than what is stated in `mem_limit` ^^
-BENTOV2_GOHAN_ES_JAVA_OPTS="ES_JAVA_OPTS=-Xms3800m -Xmx3800m"
 BENTOV2_GOHAN_ES_DATA_DIR=${BENTOV2_GOHAN_DATA_ROOT}/elasticsearch
+BENTO_GOHAN_ES_JVM_OPTIONS_DIR=${PWD}/lib/gohan/es_jvm_options
 
 # -- API-DRS Bridge
 BENTOV2_GOHAN_API_DRS_BRIDGE_HOST_DIR=${BENTOV2_GOHAN_DATA_ROOT}/api_drs_bridge
@@ -374,7 +375,7 @@ BENTOV2_GOHAN_PRIVATE_AUTHZ_URL=http://${BENTOV2_GOHAN_AUTHZ_OPA_CONTAINER_NAME}
 # Bento-Public
 
 BENTO_PUBLIC_IMAGE=ghcr.io/bento-platform/bento_public
-BENTO_PUBLIC_VERSION=0.17.3
+BENTO_PUBLIC_VERSION=0.18.0
 BENTO_PUBLIC_VERSION_DEV=${BENTO_PUBLIC_VERSION}-dev
 BENTO_PUBLIC_CONTAINER_NAME=${BENTOV2_PREFIX}-public
 BENTO_PUBLIC_NETWORK=${BENTOV2_PREFIX}-public-net
@@ -395,7 +396,7 @@ BENTO_PUBLIC_TRANSLATED=true
 BENTO_BEACON_CONTAINER_NAME=${BENTOV2_PREFIX}-beacon
 BENTO_BEACON_NETWORK=${BENTOV2_PREFIX}-beacon-net
 BENTO_BEACON_IMAGE=ghcr.io/bento-platform/bento_beacon
-BENTO_BEACON_VERSION=0.13.0
+BENTO_BEACON_VERSION=0.14.0
 BENTO_BEACON_VERSION_DEV=${BENTO_BEACON_VERSION}-dev
 BENTO_BEACON_INTERNAL_PORT=${BENTO_STD_SERVICE_INTERNAL_PORT}
 BENTO_BEACON_EXTERNAL_PORT=5000

etc/bento_post_config.bash

@@ -5,8 +5,10 @@ if [[ "$BENTO_GATEWAY_USE_TLS" == 'true' || "$BENTO_GATEWAY_USE_TLS" == '1' ]];
   KC_HTTP_ENABLED='false'
   KC_HTTPS_CERTIFICATE_FILE=/run/secrets/keycloak-cert-file
   KC_HTTPS_CERTIFICATE_KEY_FILE=/run/secrets/keycloak-cert-key-file
+  KC_PROXY='passthrough'
 else
   # Disable TLS in keycloak
   KC_HOSTNAME_STRICT_HTTPS='false'
   KC_HTTP_ENABLED='true'
+  KC_PROXY='edge'
 fi

etc/default.gohan.es.jvm.options

@@ -0,0 +1,9 @@
+#########################################################
+## FOR DEVELOPMENT ONLY
+## Xms and Xmx MUST have the same value
+## Use no more than 50% of BENTOV2_GOHAN_ES_MEM_LIM
+## Default options assume 'BENTOV2_GOHAN_ES_MEM_LIM=4G'
+#########################################################
+
+-Xms2g
+-Xmx2g

lib/auth/docker-compose.auth.yaml

@@ -7,6 +7,7 @@ services:
         condition: service_healthy
     expose:
       - ${BENTOV2_AUTH_INTERNAL_PORT}
+      - ${BENTOV2_AUTH_INTERNAL_PORT_PLAIN_HTTP}
     networks:
       - auth-net
       - auth-db-net
@@ -28,6 +29,7 @@ services:
       - KC_HTTP_ENABLED
       - KC_HTTPS_CERTIFICATE_FILE
       - KC_HTTPS_CERTIFICATE_KEY_FILE
+      - KC_PROXY
     mem_limit: ${BENTOV2_AUTH_MEM_LIM} # for mem_limit to work, make sure docker-compose is v2.4
     cpus: ${BENTOV2_AUTH_CPUS}
     cpu_shares: 512

lib/gohan/docker-compose.gohan.yaml

@@ -65,12 +65,12 @@ services:
       - BENTO_UID
       - ELASTIC_USERNAME=${BENTOV2_GOHAN_ES_USERNAME}
       - ELASTIC_PASSWORD=${BENTOV2_GOHAN_ES_PASSWORD}
-      - ${BENTOV2_GOHAN_ES_JAVA_OPTS}
       - xpack.security.enabled=true
       - discovery.type=${BENTOV2_GOHAN_ES_DISC_TYP}
       - ingest.geoip.downloader.enabled=false
     volumes:
       - ${BENTOV2_GOHAN_ES_DATA_DIR}:/usr/share/elasticsearch/data
+      - ${BENTO_GOHAN_ES_JVM_OPTIONS_DIR}:/usr/share/elasticsearch/config/jvm.options.d
     healthcheck:
       test: [ "CMD", "curl", "http://localhost:${BENTOV2_GOHAN_ES_INTERNAL_PORT_1}" ]
       timeout: ${BENTOV2_HEALTHCHECK_TIMEOUT}