Test changes for buldata web application split up

.github/workflows/integration.yml

@@ -33,6 +33,8 @@ jobs:
         mvn -B org.apache.maven.plugins:maven-dependency-plugin:3.1.2:resolve-plugins -f fhir-parent -DexcludeReactor=true -Dmaven.wagon.http.retryHandler.count=3
         mvn -B install --file fhir-parent -DskipTests -P include-fhir-igs,integration --no-transfer-progress -Dmaven.wagon.http.retryHandler.count=3
         docker build fhir-install --build-arg VERBOSE=false -t linuxforhealth/fhir-server
+        echo "Building fhir-bulkdata-server image!"
+        docker build fhir-install-bulkdata --build-arg VERBOSE=false -t linuxforhealth/fhir-bulkdata-server
     - name: free disk space
       run: |
         # create and remove a 200 MB file to make sure we have the room needed later
@@ -59,14 +61,31 @@ jobs:
         rm -fr ${it_results} 2>/dev/null
         mkdir -p ${it_results}/server-logs
         mkdir -p ${it_results}/fhir-server-test
-        containerId=$(docker ps -a | grep fhir | cut -d ' ' -f 1)
+        containerId=$(docker ps -a | grep fhir-server | cut -d ' ' -f 1)
         if [[ -z "${containerId}" ]]; then
             echo "Warning: Could not find fhir container!!!"
         else
             echo "fhir container id: $containerId"
 
             # Grab the container's console log
             docker logs $containerId  >& ${it_results}/docker-console.txt
+            echo "fhir container logs!!"
+            docker logs $containerId
+
+            echo "Gathering post-test server logs from docker container: $containerId"
+            docker cp -L $containerId:/logs ${it_results}/server-logs
+        fi
+
+        bulkDataContainerId=$(docker ps -a | grep fhir-bulkdata-server | cut -d ' ' -f 1)
+        if [[ -z "${bulkDataContainerId}" ]]; then
+            echo "Warning: Could not find bulkdata fhir container!!!"
+        else
+            echo "bulkdata fhir container id: $containerId"
+
+            # Grab the container's console log
+            docker logs $bulkDataContainerId  >& ${it_results}/docker-console_bulkdata.txt
+            echo "bulkdata fhir container logs!!"
+            docker logs $bulkDataContainerId
 
             echo "Gathering post-test server logs from docker container: $containerId"
             docker cp -L $containerId:/logs ${it_results}/server-logs

build/README.md

@@ -17,6 +17,11 @@ Once the project is built, the LinuxForHealth FHIR Server container image can be
 docker build fhir-install -t linuxforhealth/fhir-server
 ```
 
+The LinuxForHealth FHIR Bulkdata Server container image can be built from the Dockerfile under fhir-install-bulkdata:
+```sh
+docker build fhir-install-bulkdata -t linuxforhealth/fhir-bulkdata-server
+```
+
 ## Running the integration tests locally
 
 Steps to install the server to the local filesystem at the root of the project and execute the tests.
@@ -61,6 +66,7 @@ The commands are expected to work on MacOS and/or Linux.
 
 - [Docker](https://www.docker.com)
 - `linuxforhealth/fhir-server:latest` (built from the fhir-install module as described above)
+- - `linuxforhealth/fhir-bulkdata-server:latest` (built from the fhir-install-bulkdata module as described above)
 
 ### Run
 

build/audit/bin/setup-prerequisites.sh

@@ -23,6 +23,9 @@ required_build(){
 
     # Build from dockerfile
     docker build fhir-install --build-arg VERBOSE=false -t linuxforhealth/fhir-server
+
+     echo "Building fhir-bulkdata-server image!!"
+     docker build fhir-install-bulkdata --build-arg VERBOSE=false -t linuxforhealth/fhir-bulkdata-server
 }
 
 # audit_build - executes for each audit type.

build/audit/kafka/docker-compose.yml

@@ -1,4 +1,24 @@
 version: '3.8'
+volumes:
+  fhirconfig:
+    driver: local
+    driver_opts:
+      o: bind
+      type: none
+      device: ./workarea/volumes/dist/config
+  fhirconfigoverrides:
+    driver: local
+    driver_opts:
+      o: bind
+      type: none
+      device: ./workarea/volumes/dist/overrides
+  fhiruserlib:
+    driver: local
+    driver_opts:
+      o: bind
+      type: none
+      device: ./workarea/volumes/dist/userlib/
+
 services:
   zookeeper-1:
     image: confluentinc/cp-zookeeper:latest
@@ -57,16 +77,16 @@ services:
       - kafka-2
     hostname: fhir
     volumes:
-      - type: bind
-        source: ./workarea/volumes/dist/config
+      - type: volume
+        source: fhirconfig
         target: /opt/ol/wlp/usr/servers/defaultServer/config
         read_only: true
-      - type: bind
-        source: ./workarea/volumes/dist/overrides
+      - type: volume
+        source: fhirconfigoverrides
         target: /opt/ol/wlp/usr/servers/defaultServer/configDropins/overrides
         read_only: true
-      - type: bind
-        source: ./workarea/volumes/dist/userlib/
+      - type: volume
+        source: fhiruserlib
         target: /opt/ol/wlp/usr/servers/defaultServer/userlib
         read_only: true
     environment:
@@ -97,6 +117,42 @@ services:
       - ./.fhir.env
     networks:
       - fhir
+  fhir-bulkdata-server:
+    image: linuxforhealth/fhir-bulkdata-server:latest
+    depends_on:
+      - kafka-1
+      - kafka-2
+    hostname: fhirbulkdata
+    volumes:
+      - type: volume
+        source: fhirconfig
+        target: /opt/ol/wlp/usr/servers/defaultServer/config
+        read_only: true
+      - type: volume
+        source: fhirconfigoverrides
+        target: /opt/ol/wlp/usr/servers/defaultServer/configDropins/overrides
+        read_only: true
+      - type: volume
+        source: fhiruserlib
+        target: /opt/ol/wlp/usr/servers/defaultServer/userlib
+        read_only: true
+    environment:
+      - BOOTSTRAP_DB=false
+    command: bash -c "/opt/ol/wlp/bin/server run"
+    healthcheck:
+      start_period: 120s
+      interval: 10s
+      timeout: 10s
+      retries: 2
+      test: curl --max-time 5 -v -f -k -u 'fhiruser:change-password' 'https://localhost:9445/fhir-server/api/v4/healthcheck'
+    ports:
+      - 9082:9082
+      - 9445:9445
+    env_file:
+      - ./.fhir.env
+    networks:
+      - fhir
+
 networks:
   fhir:
     driver: bridge

build/azurite/docker-compose.yml

@@ -5,6 +5,30 @@ volumes:
     driver: local
   blobdata:
     driver: local
+  fhirconfig:
+    driver: local
+    driver_opts:
+      o: bind
+      type: none
+      device: ./fhir-server/config
+  fhirconfigoverrides:
+    driver: local
+    driver_opts:
+      o: bind
+      type: none
+      device: ./fhir-server/configDropins/overrides
+  fhiruserlib:
+    driver: local
+    driver_opts:
+      o: bind
+      type: none
+      device: ./fhir-server/userlib
+  fhirbulkdata:
+    driver: local
+    driver_opts:
+      o: bind
+      type: none
+      device: ./fhir-server/bulkdata
 
 services:
   postgres:
@@ -47,29 +71,24 @@ services:
     image: linuxforhealth/fhir-server:latest
     hostname: fhir
     volumes:
-      - type: bind
-        source: ./fhir-server/config
+      - type: volume
+        source: fhirconfig
         target: /opt/ol/wlp/usr/servers/defaultServer/config
         read_only: true
-      - type: bind
-        source: ./fhir-server/configDropins/overrides
+      - type: volume
+        source: fhirconfigoverrides
         target: /opt/ol/wlp/usr/servers/defaultServer/configDropins/overrides
         read_only: true
-      - type: bind
-        source: ./fhir-server/userlib
+      - type: volume
+        source: fhiruserlib
         target: /opt/ol/wlp/usr/servers/defaultServer/userlib
         read_only: true
-      - type: bind
-        source: ./fhir-server/bulkdata
+      - type: volume
+        source: fhirbulkdata
         target: /output/bulkdata
     environment:
         TRACE_SPEC: "org.linuxforhealth.fhir.*=INFO"
         TRACE_FILE: "stdout"
-        BATCH_DB_HOSTNAME: postgres
-        BATCH_DB_USER: fhirbatch
-        BATCH_DB_PASS: change-password
-        BATCH_DB_SSL: "false"
-        BATCH_DB_NAME: fhirdb
     command: bash -c "
       java -jar /opt/fhir-server/tools/fhir-persistence-blob-app-*-cli.jar
           --fhir-config-dir /opt/ol/wlp/usr/servers/defaultServer
@@ -111,6 +130,49 @@ services:
         condition: service_healthy
     networks:
       - fhir
+  fhir-bulkdata-server:
+    image: linuxforhealth/fhir-bulkdata-server:latest
+    hostname: fhirbulkdata
+    volumes:
+      - type: volume
+        source: fhirconfig
+        target: /opt/ol/wlp/usr/servers/defaultServer/config
+        read_only: true
+      - type: bind
+        source: ./fhir-bulkdata-server/configDropins/overrides
+        target: /opt/ol/wlp/usr/servers/defaultServer/configDropins/overrides
+        read_only: true
+      - type: volume
+        source: fhiruserlib
+        target: /opt/ol/wlp/usr/servers/defaultServer/userlib
+        read_only: true
+      - type: volume
+        source: fhirbulkdata
+        target: /output/bulkdata
+    environment:
+      TRACE_SPEC: "org.linuxforhealth.fhir.*=INFO"
+      TRACE_FILE: "stdout"
+      BATCH_DB_HOSTNAME: postgres
+      BATCH_DB_USER: fhirbatch
+      BATCH_DB_PASS: change-password
+      BATCH_DB_SSL: "false"
+      BATCH_DB_NAME: fhirdb
+    command: bash -c "/opt/ol/wlp/bin/server run"
+    healthcheck:
+      start_period: 32s
+      interval: 20s
+      timeout: 5s
+      retries: 3
+      # https://docs.docker.com/compose/compose-file/#variable-substitution
+      test: curl --fail -k -u 'fhiruser:change-password' 'https://localhost:9445/fhir-bulkdata-server/api/v4/healthcheck'
+    ports:
+      - 9082:9082
+      - 9445:9445
+    depends_on:
+      postgres:
+        condition: service_healthy
+    networks:
+      - fhir
 
   azurite:
     image: mcr.microsoft.com/azure-storage/azurite

build/common/copy-server-config.sh

@@ -22,12 +22,21 @@ cp ${CONFIG}/default/fhir-server-config-postgresql-minio.json ${CONFIG}/default/
 
 echo "Replacing datasource content in server configDropins..."
 OVERRIDES="fhir-server/configDropins/overrides"
+BULKDATA_OVERRIDES="fhir-bulkdata-server/configDropins/overrides"
 rm -rf ${OVERRIDES}/* 2> /dev/null
 mkdir -p ${OVERRIDES}
 
+echo "Create overrides directory for bulkdata db config..."
+rm -rf ${BULKDATA_OVERRIDES}/* 2> /dev/null
+mkdir -p ${BULKDATA_OVERRIDES}
+
 # Copy over both the postgres (default_default) and derby (tenant1_*) datasource definitions
-cp ${WORKSPACE}/fhir-server-webapp/src/main/liberty/config/configDropins/disabled/datasource-postgresql.xml ${OVERRIDES}/
-cp ${WORKSPACE}/fhir-server-webapp/src/main/liberty/config/configDropins/disabled/postgres/bulkdata.xml ${OVERRIDES}/
 cp ${WORKSPACE}/fhir-server-webapp/src/test/liberty/config/configDropins/overrides/datasource-derby.xml ${OVERRIDES}/
+cp ${WORKSPACE}/fhir-server-webapp/src/main/liberty/config/configDropins/disabled/datasource-postgresql.xml ${OVERRIDES}/
+
+echo "Copy over both the postgres (default_default) and derby (tenant1_*) datasource definitions to bulkdata_overrides..."
+cp ${WORKSPACE}/fhir-server-webapp/src/main/liberty/bulkdata/disabled/postgres/datasource-bulkdata.xml ${BULKDATA_OVERRIDES}/
+cp ${WORKSPACE}/fhir-server-webapp/src/test/liberty/config/configDropins/overrides/datasource-derby.xml ${BULKDATA_OVERRIDES}/
+cp ${WORKSPACE}/fhir-server-webapp/src/main/liberty/config/configDropins/disabled/datasource-postgresql.xml ${BULKDATA_OVERRIDES}/
 
 echo "Finished copying the server config."

build/minio/configure.sh

@@ -14,6 +14,7 @@ CONFIG="${WORKSPACE}/build/minio/fhir-server/config"
 TEST_RESOURCES="${WORKSPACE}/fhir-server-test/src/test/resources"
 
 # Set the fhir-server-config
+echo "Copying the fhir server config files..."
 cp ${CONFIG}/default/fhir-server-config-postgresql-minio.json ${CONFIG}/default/fhir-server-config.json
 
 # Enable the file-based import/export tests and set the path to the output dir