diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 6959ba39c8e..8950647ae0d 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -33,6 +33,8 @@ jobs: mvn -B org.apache.maven.plugins:maven-dependency-plugin:3.1.2:resolve-plugins -f fhir-parent -DexcludeReactor=true -Dmaven.wagon.http.retryHandler.count=3 mvn -B install --file fhir-parent -DskipTests -P include-fhir-igs,integration --no-transfer-progress -Dmaven.wagon.http.retryHandler.count=3 docker build fhir-install --build-arg VERBOSE=false -t linuxforhealth/fhir-server + echo "Building fhir-bulkdata-server image!" + docker build fhir-install-bulkdata --build-arg VERBOSE=false -t linuxforhealth/fhir-bulkdata-server - name: free disk space run: | # create and remove a 200 MB file to make sure we have the room needed later @@ -59,7 +61,7 @@ jobs: rm -fr ${it_results} 2>/dev/null mkdir -p ${it_results}/server-logs mkdir -p ${it_results}/fhir-server-test - containerId=$(docker ps -a | grep fhir | cut -d ' ' -f 1) + containerId=$(docker ps -a | grep fhir-server | cut -d ' ' -f 1) if [[ -z "${containerId}" ]]; then echo "Warning: Could not find fhir container!!!" else @@ -67,6 +69,23 @@ jobs: # Grab the container's console log docker logs $containerId >& ${it_results}/docker-console.txt + echo "fhir container logs!!" + docker logs $containerId + + echo "Gathering post-test server logs from docker container: $containerId" + docker cp -L $containerId:/logs ${it_results}/server-logs + fi + + bulkDataContainerId=$(docker ps -a | grep fhir-bulkdata-server | cut -d ' ' -f 1) + if [[ -z "${bulkDataContainerId}" ]]; then + echo "Warning: Could not find bulkdata fhir container!!!" + else + echo "bulkdata fhir container id: $containerId" + + # Grab the container's console log + docker logs $bulkDataContainerId >& ${it_results}/docker-console_bulkdata.txt + echo "bulkdata fhir container logs!!" + docker logs $bulkDataContainerId echo "Gathering post-test server logs from docker container: $containerId" docker cp -L $containerId:/logs ${it_results}/server-logs diff --git a/build/README.md b/build/README.md index bd93a3ce9f8..ebbfb2604f0 100644 --- a/build/README.md +++ b/build/README.md @@ -17,6 +17,11 @@ Once the project is built, the LinuxForHealth FHIR Server container image can be docker build fhir-install -t linuxforhealth/fhir-server ``` +The LinuxForHealth FHIR Bulkdata Server container image can be built from the Dockerfile under fhir-install-bulkdata: +```sh +docker build fhir-install-bulkdata -t linuxforhealth/fhir-bulkdata-server +``` + ## Running the integration tests locally Steps to install the server to the local filesystem at the root of the project and execute the tests. @@ -61,6 +66,7 @@ The commands are expected to work on MacOS and/or Linux. - [Docker](https://www.docker.com) - `linuxforhealth/fhir-server:latest` (built from the fhir-install module as described above) +- - `linuxforhealth/fhir-bulkdata-server:latest` (built from the fhir-install-bulkdata module as described above) ### Run diff --git a/build/audit/bin/setup-prerequisites.sh b/build/audit/bin/setup-prerequisites.sh index 71dfc728b53..055ed89b941 100644 --- a/build/audit/bin/setup-prerequisites.sh +++ b/build/audit/bin/setup-prerequisites.sh @@ -23,6 +23,9 @@ required_build(){ # Build from dockerfile docker build fhir-install --build-arg VERBOSE=false -t linuxforhealth/fhir-server + + echo "Building fhir-bulkdata-server image!!" + docker build fhir-install-bulkdata --build-arg VERBOSE=false -t linuxforhealth/fhir-bulkdata-server } # audit_build - executes for each audit type. diff --git a/build/audit/kafka/docker-compose.yml b/build/audit/kafka/docker-compose.yml index 9705d80ce3e..564ea2b912f 100644 --- a/build/audit/kafka/docker-compose.yml +++ b/build/audit/kafka/docker-compose.yml @@ -1,4 +1,24 @@ version: '3.8' +volumes: + fhirconfig: + driver: local + driver_opts: + o: bind + type: none + device: ./workarea/volumes/dist/config + fhirconfigoverrides: + driver: local + driver_opts: + o: bind + type: none + device: ./workarea/volumes/dist/overrides + fhiruserlib: + driver: local + driver_opts: + o: bind + type: none + device: ./workarea/volumes/dist/userlib/ + services: zookeeper-1: image: confluentinc/cp-zookeeper:latest @@ -57,16 +77,16 @@ services: - kafka-2 hostname: fhir volumes: - - type: bind - source: ./workarea/volumes/dist/config + - type: volume + source: fhirconfig target: /opt/ol/wlp/usr/servers/defaultServer/config read_only: true - - type: bind - source: ./workarea/volumes/dist/overrides + - type: volume + source: fhirconfigoverrides target: /opt/ol/wlp/usr/servers/defaultServer/configDropins/overrides read_only: true - - type: bind - source: ./workarea/volumes/dist/userlib/ + - type: volume + source: fhiruserlib target: /opt/ol/wlp/usr/servers/defaultServer/userlib read_only: true environment: @@ -97,6 +117,42 @@ services: - ./.fhir.env networks: - fhir + fhir-bulkdata-server: + image: linuxforhealth/fhir-bulkdata-server:latest + depends_on: + - kafka-1 + - kafka-2 + hostname: fhirbulkdata + volumes: + - type: volume + source: fhirconfig + target: /opt/ol/wlp/usr/servers/defaultServer/config + read_only: true + - type: volume + source: fhirconfigoverrides + target: /opt/ol/wlp/usr/servers/defaultServer/configDropins/overrides + read_only: true + - type: volume + source: fhiruserlib + target: /opt/ol/wlp/usr/servers/defaultServer/userlib + read_only: true + environment: + - BOOTSTRAP_DB=false + command: bash -c "/opt/ol/wlp/bin/server run" + healthcheck: + start_period: 120s + interval: 10s + timeout: 10s + retries: 2 + test: curl --max-time 5 -v -f -k -u 'fhiruser:change-password' 'https://localhost:9445/fhir-server/api/v4/healthcheck' + ports: + - 9082:9082 + - 9445:9445 + env_file: + - ./.fhir.env + networks: + - fhir + networks: fhir: driver: bridge diff --git a/build/azurite/docker-compose.yml b/build/azurite/docker-compose.yml index 73ae4073d24..299d7935923 100644 --- a/build/azurite/docker-compose.yml +++ b/build/azurite/docker-compose.yml @@ -5,6 +5,30 @@ volumes: driver: local blobdata: driver: local + fhirconfig: + driver: local + driver_opts: + o: bind + type: none + device: ./fhir-server/config + fhirconfigoverrides: + driver: local + driver_opts: + o: bind + type: none + device: ./fhir-server/configDropins/overrides + fhiruserlib: + driver: local + driver_opts: + o: bind + type: none + device: ./fhir-server/userlib + fhirbulkdata: + driver: local + driver_opts: + o: bind + type: none + device: ./fhir-server/bulkdata services: postgres: @@ -47,29 +71,24 @@ services: image: linuxforhealth/fhir-server:latest hostname: fhir volumes: - - type: bind - source: ./fhir-server/config + - type: volume + source: fhirconfig target: /opt/ol/wlp/usr/servers/defaultServer/config read_only: true - - type: bind - source: ./fhir-server/configDropins/overrides + - type: volume + source: fhirconfigoverrides target: /opt/ol/wlp/usr/servers/defaultServer/configDropins/overrides read_only: true - - type: bind - source: ./fhir-server/userlib + - type: volume + source: fhiruserlib target: /opt/ol/wlp/usr/servers/defaultServer/userlib read_only: true - - type: bind - source: ./fhir-server/bulkdata + - type: volume + source: fhirbulkdata target: /output/bulkdata environment: TRACE_SPEC: "org.linuxforhealth.fhir.*=INFO" TRACE_FILE: "stdout" - BATCH_DB_HOSTNAME: postgres - BATCH_DB_USER: fhirbatch - BATCH_DB_PASS: change-password - BATCH_DB_SSL: "false" - BATCH_DB_NAME: fhirdb command: bash -c " java -jar /opt/fhir-server/tools/fhir-persistence-blob-app-*-cli.jar --fhir-config-dir /opt/ol/wlp/usr/servers/defaultServer @@ -111,6 +130,49 @@ services: condition: service_healthy networks: - fhir + fhir-bulkdata-server: + image: linuxforhealth/fhir-bulkdata-server:latest + hostname: fhirbulkdata + volumes: + - type: volume + source: fhirconfig + target: /opt/ol/wlp/usr/servers/defaultServer/config + read_only: true + - type: bind + source: ./fhir-bulkdata-server/configDropins/overrides + target: /opt/ol/wlp/usr/servers/defaultServer/configDropins/overrides + read_only: true + - type: volume + source: fhiruserlib + target: /opt/ol/wlp/usr/servers/defaultServer/userlib + read_only: true + - type: volume + source: fhirbulkdata + target: /output/bulkdata + environment: + TRACE_SPEC: "org.linuxforhealth.fhir.*=INFO" + TRACE_FILE: "stdout" + BATCH_DB_HOSTNAME: postgres + BATCH_DB_USER: fhirbatch + BATCH_DB_PASS: change-password + BATCH_DB_SSL: "false" + BATCH_DB_NAME: fhirdb + command: bash -c "/opt/ol/wlp/bin/server run" + healthcheck: + start_period: 32s + interval: 20s + timeout: 5s + retries: 3 + # https://docs.docker.com/compose/compose-file/#variable-substitution + test: curl --fail -k -u 'fhiruser:change-password' 'https://localhost:9445/fhir-bulkdata-server/api/v4/healthcheck' + ports: + - 9082:9082 + - 9445:9445 + depends_on: + postgres: + condition: service_healthy + networks: + - fhir azurite: image: mcr.microsoft.com/azure-storage/azurite diff --git a/build/common/copy-server-config.sh b/build/common/copy-server-config.sh index 2f98d754713..9df64835421 100755 --- a/build/common/copy-server-config.sh +++ b/build/common/copy-server-config.sh @@ -22,12 +22,21 @@ cp ${CONFIG}/default/fhir-server-config-postgresql-minio.json ${CONFIG}/default/ echo "Replacing datasource content in server configDropins..." OVERRIDES="fhir-server/configDropins/overrides" +BULKDATA_OVERRIDES="fhir-bulkdata-server/configDropins/overrides" rm -rf ${OVERRIDES}/* 2> /dev/null mkdir -p ${OVERRIDES} +echo "Create overrides directory for bulkdata db config..." +rm -rf ${BULKDATA_OVERRIDES}/* 2> /dev/null +mkdir -p ${BULKDATA_OVERRIDES} + # Copy over both the postgres (default_default) and derby (tenant1_*) datasource definitions -cp ${WORKSPACE}/fhir-server-webapp/src/main/liberty/config/configDropins/disabled/datasource-postgresql.xml ${OVERRIDES}/ -cp ${WORKSPACE}/fhir-server-webapp/src/main/liberty/config/configDropins/disabled/postgres/bulkdata.xml ${OVERRIDES}/ cp ${WORKSPACE}/fhir-server-webapp/src/test/liberty/config/configDropins/overrides/datasource-derby.xml ${OVERRIDES}/ +cp ${WORKSPACE}/fhir-server-webapp/src/main/liberty/config/configDropins/disabled/datasource-postgresql.xml ${OVERRIDES}/ + +echo "Copy over both the postgres (default_default) and derby (tenant1_*) datasource definitions to bulkdata_overrides..." +cp ${WORKSPACE}/fhir-server-webapp/src/main/liberty/bulkdata/disabled/postgres/datasource-bulkdata.xml ${BULKDATA_OVERRIDES}/ +cp ${WORKSPACE}/fhir-server-webapp/src/test/liberty/config/configDropins/overrides/datasource-derby.xml ${BULKDATA_OVERRIDES}/ +cp ${WORKSPACE}/fhir-server-webapp/src/main/liberty/config/configDropins/disabled/datasource-postgresql.xml ${BULKDATA_OVERRIDES}/ echo "Finished copying the server config." diff --git a/build/minio/configure.sh b/build/minio/configure.sh index ac2a47c1d36..0e828b230a4 100755 --- a/build/minio/configure.sh +++ b/build/minio/configure.sh @@ -14,6 +14,7 @@ CONFIG="${WORKSPACE}/build/minio/fhir-server/config" TEST_RESOURCES="${WORKSPACE}/fhir-server-test/src/test/resources" # Set the fhir-server-config +echo "Copying the fhir server config files..." cp ${CONFIG}/default/fhir-server-config-postgresql-minio.json ${CONFIG}/default/fhir-server-config.json # Enable the file-based import/export tests and set the path to the output dir diff --git a/build/minio/docker-compose.yml b/build/minio/docker-compose.yml index 99ecc41ff06..6178c46b987 100644 --- a/build/minio/docker-compose.yml +++ b/build/minio/docker-compose.yml @@ -3,6 +3,31 @@ version: '3.8' volumes: pgdata: driver: local + fhirconfig: + driver: local + driver_opts: + o: bind + type: none + device: ./fhir-server/config + fhirconfigoverrides: + driver: local + driver_opts: + o: bind + type: none + device: ./fhir-server/configDropins/overrides + fhiruserlib: + driver: local + driver_opts: + o: bind + type: none + device: ./fhir-server/userlib + fhirbulkdata: + driver: local + driver_opts: + o: bind + type: none + device: ./fhir-server/bulkdata + services: postgres: @@ -45,29 +70,24 @@ services: image: linuxforhealth/fhir-server:latest hostname: fhir volumes: - - type: bind - source: ./fhir-server/config + - type: volume + source: fhirconfig target: /opt/ol/wlp/usr/servers/defaultServer/config read_only: true - - type: bind - source: ./fhir-server/configDropins/overrides + - type: volume + source: fhirconfigoverrides target: /opt/ol/wlp/usr/servers/defaultServer/configDropins/overrides read_only: true - - type: bind - source: ./fhir-server/userlib + - type: volume + source: fhiruserlib target: /opt/ol/wlp/usr/servers/defaultServer/userlib read_only: true - - type: bind - source: ./fhir-server/bulkdata + - type: volume + source: fhirbulkdata target: /output/bulkdata environment: TRACE_SPEC: "org.linuxforhealth.fhir.*=INFO" TRACE_FILE: "stdout" - BATCH_DB_HOSTNAME: postgres - BATCH_DB_USER: fhirbatch - BATCH_DB_PASS: change-password - BATCH_DB_SSL: "false" - BATCH_DB_NAME: fhirdb command: bash -c " java -jar /opt/fhir-server/tools/fhir-persistence-schema-*-cli.jar --db-type postgresql --prop db.host=postgres --prop db.port=5432 --prop db.database=fhirdb --prop user=fhiradmin --prop password=change-password @@ -106,6 +126,49 @@ services: condition: service_healthy networks: - fhir + fhir-bulkdata-server: + image: linuxforhealth/fhir-bulkdata-server:latest + hostname: fhirbulkdata + volumes: + - type: volume + source: fhirconfig + target: /opt/ol/wlp/usr/servers/defaultServer/config + read_only: true + - type: bind + source: ./fhir-bulkdata-server/configDropins/overrides + target: /opt/ol/wlp/usr/servers/defaultServer/configDropins/overrides + read_only: true + - type: volume + source: fhiruserlib + target: /opt/ol/wlp/usr/servers/defaultServer/userlib + read_only: true + - type: volume + source: fhirbulkdata + target: /output/bulkdata + environment: + TRACE_SPEC: "org.linuxforhealth.fhir.*=INFO" + TRACE_FILE: "stdout" + BATCH_DB_HOSTNAME: postgres + BATCH_DB_USER: fhirbatch + BATCH_DB_PASS: change-password + BATCH_DB_SSL: "false" + BATCH_DB_NAME: fhirdb + command: bash -c "/opt/ol/wlp/bin/server run" + healthcheck: + start_period: 32s + interval: 30s + timeout: 5s + retries: 3 + # https://docs.docker.com/compose/compose-file/#variable-substitution + test: curl --fail -k -u 'fhiruser:change-password' 'https://localhost:9445/fhir-bulkdata-server/api/v4/healthcheck' + ports: + - 9082:9082 + - 9445:9445 + depends_on: + postgres: + condition: service_healthy + networks: + - fhir nats-node1: image: nats-streaming:0.24 diff --git a/build/notifications/bin/setup-prerequisites.sh b/build/notifications/bin/setup-prerequisites.sh index 9970c009126..a0c8e0a883c 100644 --- a/build/notifications/bin/setup-prerequisites.sh +++ b/build/notifications/bin/setup-prerequisites.sh @@ -23,6 +23,9 @@ required_build(){ # Build from dockerfile docker build fhir-install --build-arg VERBOSE=false -t linuxforhealth/fhir-server + + echo "Building fhir-bulkdata-server image!!!" + docker build fhir-install-bulkdata --build-arg VERBOSE=false -t linuxforhealth/fhir-bulkdata-server } # notifications_build - executes for each notifications type. diff --git a/build/persistence/bin/setup-prerequisites.sh b/build/persistence/bin/setup-prerequisites.sh index b7e8ac8b6b2..90c1f482234 100644 --- a/build/persistence/bin/setup-prerequisites.sh +++ b/build/persistence/bin/setup-prerequisites.sh @@ -23,6 +23,9 @@ required_build(){ # Build from dockerfile docker build fhir-install --build-arg VERBOSE=false -t linuxforhealth/fhir-server + + echo "Building fhir-bulkdata-server image!!!!" + docker build fhir-install-bulkdata --build-arg VERBOSE=false -t linuxforhealth/fhir-bulkdata-server } # persistence_build - executes for each persistence build triggering the persistence layer's required steps. diff --git a/build/pre-integration-test-docker.sh b/build/pre-integration-test-docker.sh index 8157d674ea6..90c0e8cf5da 100755 --- a/build/pre-integration-test-docker.sh +++ b/build/pre-integration-test-docker.sh @@ -63,6 +63,9 @@ echo " Docker container status:" docker ps -a +echo "docker image status:" +docker images + echo "Waiting for fhir-server to complete initialization..." ${WORKSPACE}/build/common/wait-for-it.sh diff --git a/docs/src/pages/guides/FHIRServerUsersGuide.md b/docs/src/pages/guides/FHIRServerUsersGuide.md index b02fcca71a2..7bd2928146a 100644 --- a/docs/src/pages/guides/FHIRServerUsersGuide.md +++ b/docs/src/pages/guides/FHIRServerUsersGuide.md @@ -44,50 +44,60 @@ This FHIR server is intended to be a common component for providing FHIR capabil ## 2.1 Installing a new server 0. Prereqs: The LinuxForHealth FHIR Server requires Java 11 and has been tested with OpenJDK 11. To install Java on your system, we recommend downloading and installing OpenJDK 11 from https://adoptium.net/. -1. To install the LinuxForHealth FHIR Server, build or download the `fhir-install` zip installer. -The Maven build creates the zip package under `fhir-install/target`. Alternatively, releases are available from the [Releases tab](https://github.com/LinuxForHealth/fhir/releases). +1. To install the LinuxForHealth FHIR Server, build or download the `fhir-install` zip installer. To install the LinuxForHealth FHIR Bulkdata Server, build or download the `fhir-install-bulkdata` zip installer. +The Maven build creates the zip package under `fhir-install/target` and `fhir-install-bulkdata/target`. Alternatively, releases are available from the [Releases tab](https://github.com/LinuxForHealth/fhir/releases). -2. Unzip the `.zip` package into a clean directory (referred to as `fhir-installer` here): +2. Unzip the `.zip` packages into a clean directory (referred to as `fhir-installer` and `fhir-bulkdata-installer` here): ``` mkdir fhir-installer cd fhir-installer unzip fhir-server-distribution.zip + + mkdir fhir-bulkdata-installer + cd fhir-bulkdata-installer + unzip fhir-bulkdata-server-distribution.zip ``` -3. Determine an install location for the OpenLiberty server and the LinuxForHealth FHIR Server webapp. Example: `/opt/ibm/fhir-server` +3. Determine an install location for the OpenLiberty server, the LinuxForHealth FHIR Server webapp and the LinuxForHealth FHIR Bulkdata Server webapp. Example: `/opt/linuxforhealth/fhir-server` and `/opt/linuxforhealth/fhir-bulkdata-server` -4. Run the `install.sh/.bat` script to install the server: - ``` - ./fhir-server-dist/install.sh /opt/ibm/fhir-server - ``` - This step installs the OpenLiberty runtime and the LinuxForHealth FHIR Server web application. The Liberty runtime is installed in a directory called `wlp` within the installation directory that you specify. For example, in the preceding command, the root directory of the Liberty server runtime would be `/opt/ibm/fhir-server/wlp`. + 4. Run the `install.sh/.bat` script to install the servers: + ``` + install fhir-server: ./fhir-server-dist/install.sh /opt/linuxforhealth/fhir-server + install fhir-bulkdata-server: ./fhir-bulkdata-server-dist/install.sh /opt/linuxforhealth/fhir-bulkdata-server + ``` + This step installs the OpenLiberty runtime, the LinuxForHealth FHIR Server web application and the LinuxForHealth FHIR Bulkdata Server web application. The Liberty runtime is installed in a directory called `wlp` within the installation directory that you specify. For example, in the preceding command, the root directory of the Liberty server runtime would be `/opt/ibm/fhir-server/wlp`. -5. Configure the fhir-server's `server.xml` file as needed by completing the following steps: - * Configure the ports that the server listen on. The server is installed with only port 9443 (HTTPS) enabled by default. To change the port numbers, modify the values in the `httpEndpoint` element. +5. Configure the fhir-server's `server.xml` and the fhir-bulkdata-server's `server.xml` file as needed by completing the following steps: + * Configure the ports that the server listen on. The fhir-server is installed with only port 9443 (HTTPS) enabled by default. The fhir-bulkdata-server is installed with only port 9445 (HTTPS) enabled by default. To change the port numbers, modify the values in the `httpEndpoint` element. * Configure a server keystore and truststore. The LinuxForHealth FHIR Server is installed with a default keystore file that contains a single self-signed certificate for localhost. For production use, you must create and configure your own keystore and truststore files for the FHIR server deployment (that is, generate your own server certificate or obtain a trusted certificate, and then share the public key certificate with API consumers so that they can insert it into their client-side truststore). The keystore and truststore files are used along with the server's HTTPS endpoint and the FHIR server's client-certificate-based authentication protocol to secure the FHIR server's endpoint. For more information, see [Section 5.2 Keystores, truststores, and the FHIR server](#52-keystores-truststores-and-the-fhir-server). * Configure an appropriate user registry. The FHIR server is installed with a basic user registry that contains a single user named `fhiruser`. For production use, it's best to configure your own user registry. For more information about configuring user registries, see the [OpenLiberty documentation](https://openliberty.io/guides/security-intro.html#configuring-the-user-registry). To override the default fhiruser's password, one may set an Environment variable `FHIR_USER_PASSWORD` and for the fhiradmin's password one may set an Environment variable `FHIR_ADMIN_PASSWORD`. 6. Make sure that your selected database product is running and ready to accept requests as needed: - * By default, the FHIR server is installed with the JDBC persistence layer configured to use an Embedded Derby database. When using the `ibmcom/ibm-fhir-server` docker image, set the `BOOTSTRAP_DB` environment variable to `true` in order to bootstrap this database. For any other configuration, note the database host and port and, if necessary, create a user with privileges for deploying the schema. + * By default, the FHIR server and the FHIR Bulkdata server are installed with the JDBC persistence layer configured to use an Embedded Derby database. When using the `ibmcom/ibm-fhir-server` docker image, set the `BOOTSTRAP_DB` environment variable to `true` in order to bootstrap this database. For any other configuration, note the database host and port and, if necessary, create a user with privileges for deploying the schema. 7. Create and deploy the LinuxForHealth FHIR Server database schema as needed: - * By default, the FHIR server is installed with the JDBC persistence layer configured to use an Embedded Derby database. When using the `ibmcom/ibm-fhir-server` docker image, set the `BOOTSTRAP_DB` environment variable to `true` in order to bootstrap this database. For any other configuration, use the `fhir-persistence-schema` module to create and deploy the database schema. + * By default, the FHIR server and the FHIR Bulkdata server are installed with the JDBC persistence layer configured to use an Embedded Derby database. When using the `ibmcom/ibm-fhir-server` docker image, set the `BOOTSTRAP_DB` environment variable to `true` in order to bootstrap this database. For any other configuration, use the `fhir-persistence-schema` module to create and deploy the database schema. 8. Configure the `fhir-server-config.json`[1](#f1) configuration file as needed: * By default, the FHIR server is installed with the JDBC persistence layer configured to use a single-tenant Embedded Derby database. For more information, see [Section 3.3 Persistence layer configuration](#33-persistence-layer-configuration). - -9. To start and stop the server, use the Liberty server command: + * Update the 'fhirServer/bulkdata/core/api/url' with the port number of fhir-bulkdata-server. +9. To start and stop the servers, use the Liberty server command: ``` /bin/server start fhir-server /bin/server stop fhir-server + + /bin/server start fhir-bulkdata-server + /bin/server stop fhir-bulkdata-server ``` -9. After you start the server, you can verify that it's running properly by invoking the `$healthcheck` endpoint like this: +9. After you start the servers, you can verify that it's running properly by invoking the `$healthcheck` endpoints like this: ``` - curl -k -u ':' 'https://:/fhir-server/api/v4/$healthcheck' + fhir-server: curl -k -u ':' 'https://:/fhir-server/api/v4/$healthcheck' + + fhir-bulkdata-server: curl -k -u ':' 'https://:/fhir-bulkdata-server/api/v4/healthcheck' ``` where `` is one of the users configured in `server.xml` (default is `fhiruser`). diff --git a/fhir-bulkdata-webapp/pom.xml b/fhir-bulkdata-webapp/pom.xml index 66dd6959bf2..0d9d5a2daa1 100644 --- a/fhir-bulkdata-webapp/pom.xml +++ b/fhir-bulkdata-webapp/pom.xml @@ -183,7 +183,6 @@ maven-war-plugin - WebContent ${fhir.bulk.war.name} diff --git a/fhir-bulkdata-webapp/src/main/java/org/linuxforhealth/fhir/bulkdata/application/FHIRBulkDataApplication.java b/fhir-bulkdata-webapp/src/main/java/org/linuxforhealth/fhir/bulkdata/application/FHIRBulkDataApplication.java new file mode 100644 index 00000000000..a5a189a5063 --- /dev/null +++ b/fhir-bulkdata-webapp/src/main/java/org/linuxforhealth/fhir/bulkdata/application/FHIRBulkDataApplication.java @@ -0,0 +1,43 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.linuxforhealth.fhir.bulkdata.application; + +import java.util.HashSet; +import java.util.Set; +import java.util.logging.Logger; + +import javax.ws.rs.core.Application; + +import org.linuxforhealth.fhir.bulkdata.resources.HealthCheck; + +public class FHIRBulkDataApplication extends Application { + private static final Logger log = Logger.getLogger(FHIRBulkDataApplication.class.getName()); + + private Set> classes = null; + private Set singletons = null; + + public FHIRBulkDataApplication() { + + } + + @Override + public Set> getClasses() { + log.entering(this.getClass().getName(), "getClasses"); + System.out.println("##################### init bulk data rest api's"); + try { + if (classes == null) { + classes = new HashSet>(); + classes.add(HealthCheck.class); + } + return classes; + } finally { + log.exiting(this.getClass().getName(), "getClasses"); + } + } + + +} diff --git a/fhir-bulkdata-webapp/src/main/java/org/linuxforhealth/fhir/bulkdata/resources/HealthCheck.java b/fhir-bulkdata-webapp/src/main/java/org/linuxforhealth/fhir/bulkdata/resources/HealthCheck.java new file mode 100644 index 00000000000..9a22c777f02 --- /dev/null +++ b/fhir-bulkdata-webapp/src/main/java/org/linuxforhealth/fhir/bulkdata/resources/HealthCheck.java @@ -0,0 +1,87 @@ +/** + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.linuxforhealth.fhir.bulkdata.resources; + +import java.util.List; +import java.util.logging.Logger; + +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.core.Response; + +import org.linuxforhealth.fhir.exception.FHIROperationException; +import org.linuxforhealth.fhir.model.resource.OperationOutcome; +import org.linuxforhealth.fhir.model.resource.OperationOutcome.Issue; +import org.linuxforhealth.fhir.model.type.code.IssueSeverity; +import org.linuxforhealth.fhir.persistence.FHIRPersistence; +import org.linuxforhealth.fhir.persistence.helper.FHIRPersistenceHelper; +import org.linuxforhealth.fhir.persistence.helper.FHIRTransactionHelper; + +/** + * Resource class for the health check of bulkdata web application. + */ +@Path("/") +public class HealthCheck { + + + public HealthCheck() throws Exception { + super(); + } + + private static final Logger log = java.util.logging.Logger.getLogger(HealthCheck.class.getName()); + + FHIRPersistence fhirPersistence; + + /** + * This method validates the health check of bulkdata web application. + * @return Response - The health check response status + * @throws FHIROperationException + */ + @GET + @Path("healthcheck") + public Response healthCheck() throws FHIROperationException { + log.entering(this.getClass().getName(), "healthCheck()"); + try { + FHIRPersistenceHelper fhirPersistenceHelper = new FHIRPersistenceHelper(null); + fhirPersistence = fhirPersistenceHelper.getFHIRPersistenceImplementation(); + FHIRTransactionHelper txn = new FHIRTransactionHelper(fhirPersistence.getTransaction()); + txn.begin(); + try { + OperationOutcome operationOutcome = fhirPersistence.getHealth(); + checkOperationOutcome(operationOutcome); + return Response.status(Response.Status.OK).build(); + } catch (Throwable t) { + txn.setRollbackOnly(); + throw t; + } finally { + txn.end(); + } + } catch (FHIROperationException e) { + throw e; + } catch (Throwable t) { + throw new FHIROperationException("Unexpected error occurred while processing request for bulkdata healthcheck operation : " + getCausedByMessage(t), t); + } finally { + log.exiting(this.getClass().getName(), "healthCheck()"); + } + } + + private void checkOperationOutcome(OperationOutcome oo) throws FHIROperationException { + List issues = oo.getIssue(); + for (Issue issue : issues) { + IssueSeverity severity = issue.getSeverity(); + if (severity != null && (IssueSeverity.ERROR.getValue().equals(severity.getValue()) + || IssueSeverity.FATAL.getValue().equals(severity.getValue()))) { + throw new FHIROperationException("The persistence layer reported one or more issues").withIssue(issues); + } + } + } + + private String getCausedByMessage(Throwable throwable) { + return throwable.getClass().getName() + ": " + throwable.getMessage(); + } + + +} diff --git a/fhir-bulkdata-webapp/src/main/webapp/WEB-INF/web.xml b/fhir-bulkdata-webapp/src/main/webapp/WEB-INF/web.xml index 9efbd472e23..9f4d6327e00 100644 --- a/fhir-bulkdata-webapp/src/main/webapp/WEB-INF/web.xml +++ b/fhir-bulkdata-webapp/src/main/webapp/WEB-INF/web.xml @@ -1,6 +1,24 @@ - - IBM FHIR Server - Bulk Data WebAPP - + + IBM FHIR Server - Bulk Data WebAPP + + FHIRRestServlet + com.ibm.websphere.jaxrs.server.IBMRestServlet + + javax.ws.rs.Application + org.linuxforhealth.fhir.bulkdata.application.FHIRBulkDataApplication + + 1 + + + FHIRRestServlet + /* + + + CLIENT-CERT + diff --git a/fhir-install-bulkdata/.gitignore b/fhir-install-bulkdata/.gitignore new file mode 100644 index 00000000000..6978ab38556 --- /dev/null +++ b/fhir-install-bulkdata/.gitignore @@ -0,0 +1 @@ +.~lock.* diff --git a/fhir-install-bulkdata/Dockerfile b/fhir-install-bulkdata/Dockerfile new file mode 100644 index 00000000000..c02d026943f --- /dev/null +++ b/fhir-install-bulkdata/Dockerfile @@ -0,0 +1,73 @@ +# ---------------------------------------------------------------------------- +# (C) Copyright IBM Corp. 2022 +# +# SPDX-License-Identifier: Apache-2.0 +# ---------------------------------------------------------------------------- +# Stage: Base + +FROM openliberty/open-liberty:22.0.0.10-kernel-slim-java11-openj9-ubi as base + +USER root +RUN yum install -y unzip +RUN install -d -o 1001 /opt/fhir-bulkdata-server +USER 1001 + +COPY target/fhir-bulkdata-server-distribution.zip /tmp/ +RUN unzip -qq /tmp/fhir-bulkdata-server-distribution.zip -d /tmp && \ + /tmp/fhir-bulkdata-server-dist/install.sh /opt/ol && \ + mv /tmp/fhir-bulkdata-server-dist/tools /opt/fhir-bulkdata-server/tools +COPY src/main/docker/fhir-bulkdata-server/bootstrap.properties /opt/ol/wlp/usr/servers/defaultServer/ +COPY src/main/docker/fhir-bulkdata-server/bootstrap.sh /opt/fhir-bulkdata-server/ +# ---------------------------------------------------------------------------- +# Stage: Runnable + +FROM openliberty/open-liberty:22.0.0.10-kernel-slim-java11-openj9-ubi + +ARG VERBOSE=true +ARG FHIR_SERVER_VERSION=5.0.0-SNAPSHOT + +# The following labels are required: +LABEL name='LinuxForHealth FHIR Bulkdata Server' +LABEL version="$FHIR_SERVER_VERSION" +LABEL summary="LinuxForHealth FHIR Bulkdata Server with OpenJ9 and UBI 8" +LABEL description="The LinuxForHealth FHIR Bulkdata Server is a Standalone web application to process bulk data requests as JSR352 Java Batch jobs" + +ENV FHIR_CONFIG_HOME=/opt/ol/wlp/usr/servers/defaultServer \ + WLP_LOGGING_CONSOLE_SOURCE=message,trace,accessLog,ffdc,audit \ + WLP_LOGGING_CONSOLE_LOGLEVEL=info \ + WLP_LOGGING_CONSOLE_FORMAT=SIMPLE \ + WLP_LOGGING_MESSAGE_SOURCE="" \ + WLP_LOGGING_MESSAGE_FORMAT=JSON \ + TRACE_FILE=stdout \ + TRACE_FORMAT=BASIC + +COPY target/LICENSE /licenses/ + +COPY --chown=1001:0 --from=base /opt/ol/wlp/usr/servers/defaultServer/server.xml /opt/ol/wlp/usr/servers/defaultServer/ +COPY --chown=1001:0 --from=base /opt/ol/wlp/usr/servers/defaultServer/configDropins /opt/ol/wlp/usr/servers/defaultServer/configDropins + +RUN features.sh + +COPY --chown=1001:0 --from=base /opt/ol/wlp/usr /opt/ol/wlp/usr + +RUN configure.sh && \ + mkdir -p /output/bulkdata + +COPY --chown=1001:0 --from=base /opt/fhir-bulkdata-server /opt/fhir-bulkdata-server + +RUN mkdir -p /config/configDropins/overrides && \ + chmod -R 775 /config/configDropins/overrides && \ + chmod -R 775 /opt/ol/wlp/usr/servers/defaultServer/configDropins/defaults + +# This block ensures the latest software is picked up. +USER root +RUN yum update -y && \ + yum clean all && \ + rm -rf /var/cache/yum +USER 1001 + +# Set the working directory to the liberty defaultServer +WORKDIR ${FHIR_CONFIG_HOME} + +ENTRYPOINT ["/opt/fhir-bulkdata-server/bootstrap.sh"] +CMD ["/opt/ol/wlp/bin/server", "run", "defaultServer"] \ No newline at end of file diff --git a/fhir-install-bulkdata/README.md b/fhir-install-bulkdata/README.md new file mode 100644 index 00000000000..495597b6a49 --- /dev/null +++ b/fhir-install-bulkdata/README.md @@ -0,0 +1,45 @@ +# LinuxForHealth FHIR Server Install + +Running the LinuxForHealth FHIR Bulkdata Server as a Docker container. + +## Prerequisites + +- [Docker](http://docker.com) +- Updated fhir-install-bulkdata/target/fhir-bulkdata-server-distribution.zip + +## Build + +Using Docker Terminal, access the fhir-install-bulkdata directory and run: + +```sh +docker build -t fhir-bulkdata-server . +``` + +## Run + +Once the image is built, start it with: + +```sh +docker run -it -p 9445:9445 --name fhir-bulkdata-server --rm fhir-bulkdata-server +``` + +## Test + +Once the fhir-server is ready, you can test it by accessing: https://localhost:9444/fhir-bulkdata-server/api/v4/healthcheck + +For example: + +```sh +curl -k -i -u 'fhiruser:change-password' 'https://localhost:9445/fhir-bulkdata-server/api/v4/healthcheck' +``` + +This request makes a connection to the configured database (embedded Derby by default) and a successful response will return with: + +``` +HTTP/2 200 +date: #current_date# +content-length: 0 +content-language: en-IN +``` + +FHIR® is the registered trademark of HL7 and is used with the permission of HL7. diff --git a/fhir-install-bulkdata/pom.xml b/fhir-install-bulkdata/pom.xml new file mode 100644 index 00000000000..60cd69875ed --- /dev/null +++ b/fhir-install-bulkdata/pom.xml @@ -0,0 +1,213 @@ + + 4.0.0 + + org.linuxforhealth.fhir + fhir-parent + 5.0.0-SNAPSHOT + ../fhir-parent + + + fhir-install-bulkdata + + pom + + + + ${project.groupId} + fhir-bulkdata-webapp + ${project.version} + war + + + io.openliberty + openliberty-runtime + zip + + + org.apache.derby + derby + + + org.apache.derby + derbyclient + + + org.apache.derby + derbyshared + + + org.apache.derby + derbytools + + + org.postgresql + postgresql + + + ${project.groupId} + fhir-persistence-schema + ${project.version} + cli + + + ${project.groupId} + fhir-persistence-blob-app + ${project.version} + cli + + + + + + + maven-antrun-plugin + + + package + + + + + + + + + + + run + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + fhir-bulkdata-server-dist + + single + + package + + fhir-bulkdata-server-distribution + false + true + + src/main/assembly/distribution.xml + + + + + + + + + + + docker-build + + + !windows + + + + + + com.spotify + dockerfile-maven-plugin + + + default-cli + none + + build + + + + tag-version + none + + tag + + + ${project.version} + + + + push-latest + none + + push + + + latest + + + + push-version + none + + push + + + ${project.version} + + + + + linuxforhealth/fhir-bulkdata-server + + + + + + + include-fhir-igs + + false + + + + ${project.groupId} + fhir-ig-us-core + ${project.version} + + + ${project.groupId} + fhir-ig-carin-bb + ${project.version} + + + ${project.groupId} + fhir-ig-mcode + ${project.version} + + + ${project.groupId} + fhir-ig-davinci-hrex + ${project.version} + + + ${project.groupId} + fhir-ig-davinci-pdex + ${project.version} + + + ${project.groupId} + fhir-ig-davinci-pdex-plan-net + ${project.version} + + + ${project.groupId} + fhir-ig-davinci-pdex-formulary + ${project.version} + + + + + diff --git a/fhir-install-bulkdata/src/main/assembly/distribution.xml b/fhir-install-bulkdata/src/main/assembly/distribution.xml new file mode 100644 index 00000000000..098db899dd9 --- /dev/null +++ b/fhir-install-bulkdata/src/main/assembly/distribution.xml @@ -0,0 +1,110 @@ + + + fhir-bulkdata-distribution + fhir-bulkdata-server-dist + true + + zip + + + + ../fhir-server-webapp/src/main/liberty/config + artifacts/servers/defaultServer + + server.xml + configDropins/disabled/datasource-postgresql.xml + configDropins/disabled/jwtRS.xml + configDropins/defaults/openapi.xml + configDropins/defaults/monitoring.xml + + 0755 + + + ../fhir-server-webapp/src/main/liberty/bulkdata/disabled + artifacts/servers/defaultServer/configDropins/disabled + 0755 + + + ../fhir-server-webapp/src/main/liberty/bulkdata/disabled/derby + artifacts/servers/defaultServer/configDropins/overrides + 0755 + + + src/main/resources/scripts + + 0755 + + + + + ../fhir-server-webapp/src/main/liberty/bulkdata/bulkdata.xml + artifacts/servers/defaultServer + server.xml + 0755 + + + src/main/resources/docs/README.txt + artifacts/servers/defaultServer/docs + 0755 + + + ../fhir-bulkdata-webapp/target/fhir-bulkdata-webapp.war + artifacts/servers/defaultServer/apps + 0755 + + + + + false + 0755 + 0755 + + io.openliberty:openliberty-runtime:* + + + + false + artifacts/servers/defaultServer/userlib + 0755 + 0755 + + + ${project.groupId}:fhir-ig-* + + + + false + artifacts/shared/resources/lib/derby + 0755 + 0755 + + org.apache.derby:derby + org.apache.derby:derbyclient + org.apache.derby:derbyshared + org.apache.derby:derbytools + + + + false + artifacts/shared/resources/lib/postgresql + 0755 + 0755 + + org.postgresql:postgresql + + + + false + tools + 0755 + 0755 + + ${project.groupId}:fhir-persistence-schema:jar:cli + ${project.groupId}:fhir-persistence-blob-app:jar:cli + + + + diff --git a/fhir-install-bulkdata/src/main/docker/fhir-bulkdata-server/bootstrap.properties b/fhir-install-bulkdata/src/main/docker/fhir-bulkdata-server/bootstrap.properties new file mode 100644 index 00000000000..9cec98b7771 --- /dev/null +++ b/fhir-install-bulkdata/src/main/docker/fhir-bulkdata-server/bootstrap.properties @@ -0,0 +1,6 @@ +# disable writing to trace.log by only sending trace data to console +com.ibm.ws.logging.trace.format=BASIC +com.ibm.ws.logging.trace.file.name=stdout + +# Required to avoid conflict between CXF and FHIR interpretation of the _type query param +jaxrs.cxf.use.noop.requestPreprocessor=true \ No newline at end of file diff --git a/fhir-install-bulkdata/src/main/docker/fhir-bulkdata-server/bootstrap.sh b/fhir-install-bulkdata/src/main/docker/fhir-bulkdata-server/bootstrap.sh new file mode 100755 index 00000000000..24d272a8bd4 --- /dev/null +++ b/fhir-install-bulkdata/src/main/docker/fhir-bulkdata-server/bootstrap.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash + +# ---------------------------------------------------------------------------- +# (C) Copyright IBM Corp. 2022 +# +# SPDX-License-Identifier: Apache-2.0 +# ---------------------------------------------------------------------------- + +############################################################################## +# Description: +# This script is the entrypoint used by the fhir-server docker image, and +# optionally bootstraps a derby database prior to running the IBM FHIR Server. + +set -e -o pipefail + +############################################################################## +# The global variables used are: + +SCRIPT_NAME="$(basename ${BASH_SOURCE[0]})" + +CUR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +FHIR_PERSISTENCE_SCHEMA_CLI_LOCATION="/opt/fhir-bulkdata-server/tools" + +PERFORM_BOOTSTRAP_DB=${BOOTSTRAP_DB} +[ -z "${BOOTSTRAP_DB}" ] && PERFORM_BOOTSTRAP_DB="false" + +############################################################################## +# Helper Functions + +# info - - local function to echo info message +# ARGUMENTS: +# String of message +function info { + echo "${SCRIPT_NAME} - [INFO]: $(date +"%Y-%m-%d_%T") - ${1}" +} + +# _call_derby_db - local function to call derby database +# ARGUMENTS: +# String of additional parameters +function _call_derby_db { + /opt/java/openjdk/bin/java -jar ${FHIR_PERSISTENCE_SCHEMA_CLI_LOCATION}/fhir-persistence-schema-*-cli.jar \ + --prop "db.create=Y" \ + --prop "db.database=/output/derby/fhirDB" \ + --db-type derby \ + ${1} 2>&1 +} + +# _bootstrap_db - local function to perform database bootstrapping +function _bootstrap_db { + if [ "$PERFORM_BOOTSTRAP_DB" = "true" ] + then + info "Performing Derby database bootstrapping" + _call_derby_db "--update-schema" + info "Finished Derby database bootstrapping" + else + info "Skipping Derby database bootstrapping" + fi +} + +############################################################################## +# Script logic: + +info "Current directory: $CUR_DIR" +_bootstrap_db + +# Pass it on to the Liberty entrypoint +exec /opt/ol/helpers/runtime/docker-server.sh "$@" + +# EOF diff --git a/fhir-install-bulkdata/src/main/docker/fhir-schematool/.gitignore b/fhir-install-bulkdata/src/main/docker/fhir-schematool/.gitignore new file mode 100644 index 00000000000..e68fdde7bad --- /dev/null +++ b/fhir-install-bulkdata/src/main/docker/fhir-schematool/.gitignore @@ -0,0 +1,3 @@ +target/ +fhirschema.log +out.log \ No newline at end of file diff --git a/fhir-install-bulkdata/src/main/docker/fhir-schematool/DESIGN.md b/fhir-install-bulkdata/src/main/docker/fhir-schematool/DESIGN.md new file mode 100644 index 00000000000..2650bad8bd6 --- /dev/null +++ b/fhir-install-bulkdata/src/main/docker/fhir-schematool/DESIGN.md @@ -0,0 +1,59 @@ +# **Design** + +The LinuxForHealth FHIR Schema Tool provides an Docker image that wraps the `fhir-persistence-schema` executable jar. The tool is state machine which take a single configuration to establish a current and running state of the LinuxForHealth FHIR Server. + +The design is such that it COULD, but does not support multiple input files at this time. + +When the container is started, the container process one of two flows: + +* Schema Onboarding - creates and updates a schema and apply grants +* Schema Offboarding - drops a schema + +**Schema Onboarding Flow** + +The schema onboarding flow setup add a new tenant or update the database to the latest schema levels. + +1. Check if set to **skip**: + 1. If `SKIP['true']`, stop and exit cleanly. + 1. If `SKIP['false']`, proceed. +1. Check if **BEHAVIOR** set to ONBOARD: + 1. If not `BEHAVIOR['ONBOARD']`, skip. + 1. If `BEHAVIOR['ONBOARD']`, proceed. +1. Create the database configuration file +1. Check connectivity: + 1. If connectivity fails, stop and exit in error. + 1. If connectivity succeeds, proceed. +1. Create the schema +1. Update the Schema +1. Grant permissions to the LinuxForHealth FHIR Server database user + +**Schema Offboarding Flow** + +The schema offboarding flow offboards the current schema, while preserving the schema for multiple tenants (if supported by the database type). + +1. Check if set to **SKIP**: + 1. If `SKIP['true']`, stop and exit cleanly. + 1. If `SKIP['false']`, proceed. +1. Check if **BEHAVIOR** set to OFFBOARD: + 1. If not `BEHAVIOR['OFFBOARD']`, skip. + 1. If `BEHAVIOR['OFFBOARD']`, proceed. + 1. Drop FHIR Schema + 1. Drop Java Batch + 1. Drop OAuth + +# **Implementation Details** + +## Configuration + +The configuration is drived primarily from a working directory, and in alternative circumstances backed by an Environment variable with the input. + +The configuration data is mounted to `/fhir-schematool/workarea/input`. + +## Logging +1. Logging - The logging is to standard error and standard out. + - The logs data is put into the `/fhir-schematool/workarea/output` folder. +2. Must Gather Flow - Each execution generates a new set of log files to the `/fhir-schematool/workarea/output` folder. + +## Docker +The design is to use as few layers as possible starting with `ibmsemeruruntime/open-11-jdk:ubi_min-jre`. +The build uses multiple stages to avoid a bloated image after the necessary updates. \ No newline at end of file diff --git a/fhir-install-bulkdata/src/main/docker/fhir-schematool/DEVELOPMENT.md b/fhir-install-bulkdata/src/main/docker/fhir-schematool/DEVELOPMENT.md new file mode 100644 index 00000000000..ebd3d67a196 --- /dev/null +++ b/fhir-install-bulkdata/src/main/docker/fhir-schematool/DEVELOPMENT.md @@ -0,0 +1,61 @@ +# Development + +The document helps developers setup the development environment for the LinuxForHealth FHIR Server Schema Tool. + +1. Build the `fhir-examples` and `fhir-parent` + +``` shell +export WORKSPACE=`pwd` +cd fhir-install-bulkdata/src/main/docker/fhir-schematool +mvn clean install -DskipTests -f ${WORKSPACE}/fhir-examples/ +mvn clean install -DskipTests -f ${WORKSPACE}/fhir-parent/ +``` + +2. Change to the `fhir-install-bulkdata/src/main/docker/fhir-schematool` directory + +3. Prep the contents + +``` shell +mkdir -p target +cp ${WORKSPACE}/fhir-persistence-schema/target/fhir-persistence-schema-*-cli.jar target/ +cp ${WORKSPACE}/LICENSE target/LICENSE +``` + +4. Build the schema tool. + +``` shell +docker build --tag linuxforhealth/fhir-schematool:latest . +``` + +or + +``` shell +docker build --build-arg FHIR_VERSION=5.0.0 -t linuxforhealth/fhir-schematool:5.0.0 . +``` + +5. For Postgres, run. (You may have to edit the corresponding examples) + +* onboard +``` shell +time docker run --env ENV_TOOL_INPUT=`cat examples/postgres/persistence-offboard-example.json |base64` linuxforhealth/fhir-schematool:latest | tee out.log +``` + +* offboard +``` shell +time docker run --env ENV_TOOL_INPUT=`cat examples/postgres/persistence-onboard-example.json |base64` linuxforhealth/fhir-schematool:latest | tee out.log +``` + +6. Confirm you see your changes work, and the Deployment works. + + * Schema is created, or deleted + * No Error Logs (at least unexpected) + * Output is as expected + +# Run a Shell Check + +Sanity check of the Shell Script + +``` +brew install shellcheck +shellcheck run.sh +``` diff --git a/fhir-install-bulkdata/src/main/docker/fhir-schematool/Dockerfile b/fhir-install-bulkdata/src/main/docker/fhir-schematool/Dockerfile new file mode 100644 index 00000000000..6ab9b7a16b5 --- /dev/null +++ b/fhir-install-bulkdata/src/main/docker/fhir-schematool/Dockerfile @@ -0,0 +1,66 @@ +# ---------------------------------------------------------------------------- +# (C) Copyright IBM Corp. 2022 +# +# SPDX-License-Identifier: Apache-2.0 +# ---------------------------------------------------------------------------- +# Stage: Base + +# IBM Semeru Runtimes provides Non-official docker images as part of this repo. These are maintained by IBM. +# The link to Semeru is at https://hub.docker.com/r/ibmsemeruruntime/open-11-jdk +FROM ibmsemeruruntime/open-11-jdk:ubi_min-jre as base + +# Create the base working directory +RUN mkdir -p /opt/schematool/workarea + +# Copy in the relevant artifacts in a single command +COPY ./run.sh ./target/fhir-persistence-schema-*-cli.jar ./target/LICENSE /opt/schematool/ + +RUN curl -L -o /opt/schematool/jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 && \ + chmod +x /opt/schematool/jq + +# ---------------------------------------------------------------------------- +# Stage: Runnable + +FROM registry.access.redhat.com/ubi8/ubi-minimal + +ARG FHIR_VERSION=4.8.0 + +# The following labels are required: +LABEL name='IBM FHIR Schema Tool' +LABEL vendor='IBM' +LABEL version="$FHIR_VERSION" +LABEL summary="Image for IBM FHIR Server Schema Tool with OpenJ9 and UBI 8" +LABEL description="The IBM FHIR Server Schema Tool is designed to create and update the IBM FHIR Server's schema." + +# Environment variables +ENV SKIP false +ENV TOOL_INPUT false + +ENV LANG='en_US.UTF-8' +ENV LANGUAGE='en_US:en' +ENV LC_ALL='en_US.UTF-8' +ENV TZ 'UTC' + +WORKDIR /opt/schematool + +COPY --chown=1001:0 --from=base /opt/ /opt/ + +RUN chmod -R 755 /opt/schematool/run.sh && \ + mkdir -p /licenses && \ + mv /opt/schematool/LICENSE /licenses && \ + microdnf update -y && \ + microdnf install -y nc tzdata openssl curl ca-certificates fontconfig glibc-langpack-en gzip tar findutils shadow-utils && \ + groupadd -r fhirschemaadmin -g 1001 && \ + useradd -u 1001 -r -g 1001 -m -d /opt/schematool/home -s /sbin/nologin fhirschemaadmin && \ + chmod -R 755 /opt/schematool && \ + rm -rf /var/cache/yum && \ + rm -f /@System.solv && \ + microdnf clean all && \ + rm -rf /var/tmp/* && \ + rm -rf /tmp/* && \ + mkdir -p /opt/schematool/workarea && \ + chmod -R 775 /opt/schematool/workarea + +USER 1001 + +ENTRYPOINT ["/opt/schematool/run.sh"] \ No newline at end of file diff --git a/fhir-install-bulkdata/src/main/docker/fhir-schematool/README.md b/fhir-install-bulkdata/src/main/docker/fhir-schematool/README.md new file mode 100644 index 00000000000..1f028ed1a9c --- /dev/null +++ b/fhir-install-bulkdata/src/main/docker/fhir-schematool/README.md @@ -0,0 +1,194 @@ +# IBM FHIR Server - Schema Tool + +The LinuxForHealth FHIR Server Schema Tool is designed to create and update the LinuxForHealth FHIR Server's schema idempotently. + +The tool supports the following flows in one image: + +* Onboarding - create SQL schemas, populate the SQL objects, grant permissions to the database user. +* Offboarding - removes the schema +* Custom - executes a single fhir-persistence-schema cli action +* Debug - outputs debug information + +# **Use** + +The solution should set the variables as Environment Variables or mounted into the workarea folder as a volume which contains the persistence.json file. + +## **Environment Variables** +The following environment variables: + +| Name | Purpose | +|----------------|----------| +| ENV_SKIP | Stop the container from making any changes, and passes through with a successful state change: `[empty\|true\|false]`| +| ENV_TOOL_INPUT | Encoded String, in most circumstances base64 encoded or well escaped text, of the json| +| ENV_TOOL_DEBUG | Flags the debug | + +## Configuration Commandline +The following is read from the properties file: + +| Name | Purpose | +|------------|----------| +| tool.behavior | Switches from the Onboarding BEHAVIOR to Offboarding BEHAVIOR flow: `[onboard\|offboard\|debug\|custom]`. | +| db.type | The database type - `[postgresql]` | +| db.host | The database server hostname| +| db.port | The database server port| +| db.database | The name of the database| +| user | A username with connect and admin permissions on the target database| +| password | The user password for connecting to the database| +| ssl | true or anything else, true triggers JDBC to use ssl, an example --prop ssl=true (postgres) | +| tenant.name | the tenant name is typically default | +| schema.name.oauth | uses the default or custom | +| schema.name.fhir | defaults to fhirdata | +| schema.name.batch | uses the default or custom | +| grant.to | grants access to a specific user (which is going to run the application) | +| sslmode | For Postgres, you can set verify-full | +| sslrootcert | For Postgres, you must set as /opt/schematool/workarea/db.cert | +| db.cert | For Postgres, you must set as a base64 encoding of the certificate | + +Further, any property supported by the [fhir-persistence-schema](https://github.com/LinuxForHealth/FHIR/blob/main/fhir-persistence-schema/README.md) module is put into the file and mounted to the system. + +## Configuration file - persistence.json +The configuration file is as follows in the examples configuration. + +``` +{ + "persistence": [ + { + "db": { + "host": "172.17.0.3", + "port": "5432", + "database": "fhirdb", + "user": "postgres", + "password": "change-password", + "type": "postgresql", + "ssl": "false", + "certificate_base64": "empty" + }, + "schema": { + "fhir": "fhirdata", + "batch": "", + "oauth": "" + }, + "grant": "fhirserver", + "behavior": "onboard" + } + ] +} +``` + +You can run locally using: + +*Mac* + +``` +docker run --env ENV_TOOL_INPUT=$(cat persistence.json | base64) linuxforhealth/fhir-schematool:latest +``` + +*Linux* + +``` +docker run --env ENV_TOOL_INPUT=$(cat persistence.json | base64 -w 0) linuxforhealth/fhir-schematool:latest +``` + +An example volume mount: + +``` + volumeMounts: + - name: binding + mountPath: "/opt/schematool/workarea" + readOnly: true + volumes: + - name: binding + secret: + secretName: binding-persistence + items: + - key: binding + path: persistence.json +``` + +# Running the Tool + +## Running: Debug Behavior + +The debug behavior outputs the details of the running image: + +Using an encoded persistence.json + +``` shell +docker run linuxforhealth/fhir-schematool:latest --tool.behavior=debug +``` + +Output +``` +run.sh - [INFO]: 2020-11-09_21:39:27 - The files included with the tool are: +total 37M +drwxr-xr-x 1 root root 4.0K Nov 9 21:39 . +drwxr-xr-x 1 root root 4.0K Nov 9 20:00 .. +-rw-r--r-- 1 root root 37M Nov 6 20:33 fhir-persistence-schema-4.5.0-SNAPSHOT-cli.jar +-rwxr-xr-x 1 root root 632 Nov 9 21:39 jq +-rwxr-xr-x 1 root root 15K Nov 9 21:38 run.sh +drwxr-xr-x 2 root root 4.0K Nov 9 19:23 workarea +The OpenSSL version is: +OpenSSL 1.1.1g FIPS 21 Apr 2020 +``` + +## Running: Offboard Behavior + +Using an encoded persistence.json + +*Mac* + +``` +docker run --env ENV_TOOL_INPUT=`cat examples/postgres/persistence-offboard-example.json |base64` \ + linuxforhealth/fhir-schematool:latest | tee out.log +``` + +*Linux* + +``` +docker run --env ENV_TOOL_INPUT=`cat examples/postgres/persistence-offboard-example.json |base64 -w 0` \ + linuxforhealth/fhir-schematool:latest | tee out.log +``` + +Using arguments on the commandline + +``` shell +docker run linuxforhealth/fhir-schematool:latest --tool.behavior=offboard --db.host=172.17.0.3 \ + --db.port=50000 --user=postgres --password=change-password --db.database=fhirdb \ + --sslConnection=false --db.type=postgresql --schema.name.fhir=fhirdata --grant.to=fhirserver \ + 2>&1 | tee out.log +``` + +## Running: Onboard Behavior + +Using an encoded persistence.json + +*Mac* + +``` +docker run --env ENV_TOOL_INPUT=`cat examples/postgres/persistence-onboard-example.json |base64` \ + linuxforhealth/fhir-schematool:latest | tee out.log +``` + +*Linux* + +``` +docker run --env ENV_TOOL_INPUT=`cat examples/postgres/persistence-onboard-example.json |base64 -w 0` \ + linuxforhealth/fhir-schematool:latest | tee out.log +``` + +Using arguments on the commandline + +``` shell +docker run linuxforhealth/fhir-schematool:latest --tool.behavior=onboard --db.host=172.17.0.3 \ + --db.port=50000 --user=postgres --password=change-password --db.database=fhirdb \ + --sslConnection=false --db.type=postgresql --schema.name.fhir=fhirdata --grant.to=fhirserver \ + 2>&1 | tee out.log +``` + +************ +# **License** + +The LinuxForHealth FHIR Server - Schema Tool is licensed under the Apache 2.0 license. Full license text is available at [LICENSE](https://github.com/LinuxForHealth/FHIR/blob/main/LICENSE). + +FHIR® is the registered trademark of HL7 and is used with the permission of HL7. Use of the FHIR trademark does not constitute endorsement of this product by HL7. +IBM and the IBM logo are trademarks of International Business Machines Corporation, registered in many jurisdictions worldwide. Other product and service names might be trademarks of IBM or other companies. A current list of IBM trademarks is available on [https://ibm.com/trademark](https://ibm.com/trademark). diff --git a/fhir-install-bulkdata/src/main/docker/fhir-schematool/build.sh b/fhir-install-bulkdata/src/main/docker/fhir-schematool/build.sh new file mode 100644 index 00000000000..d5fee601f84 --- /dev/null +++ b/fhir-install-bulkdata/src/main/docker/fhir-schematool/build.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +# ---------------------------------------------------------------------------- +# (C) Copyright IBM Corp. 2022 +# +# SPDX-License-Identifier: Apache-2.0 +# ---------------------------------------------------------------------------- + +############################################################################## + +if [ -z "${WORKSPACE}" ] +then + echo "The WORKSPACE is not set" + exit 10; +fi + +if [ -z "${BUILD_ID}" ] +then + echo "The Build ID is not set" + exit 11; +fi + +# docker login is already complete at this point. + +pushd $(pwd) + +cd ${WORKSPACE}/fhir-install/src/main/docker/fhir-schematool/ + +# Copy the files over +mkdir -p target/ +cp ${WORKSPACE}/fhir-persistence-schema/target/fhir-persistence-schema-*cli.jar target/ +cp ${WORKSPACE}/LICENSE target/ + +docker build --build-arg FHIR_VERSION=${BUILD_ID} -t linuxforhealth/fhir-schematool:latest . +DOCKER_IMAGE=$(docker images --filter=reference='linuxforhealth/fhir-schematool:latest' --format "{{.ID}}") +echo "Docker Image is: ${DOCKER_IMAGE}" + +docker tag ${DOCKER_IMAGE} ghcr.io/linuxforhealth/fhir-schematool:${BUILD_ID} +docker tag ${DOCKER_IMAGE} ghcr.io/linuxforhealth/fhir-schematool:latest +docker push ghcr.io/linuxforhealth/fhir-schematool:${BUILD_ID} +docker push ghcr.io/linuxforhealth/fhir-schematool:latest + +popd > /dev/null + +# EOF \ No newline at end of file diff --git a/fhir-install-bulkdata/src/main/docker/fhir-schematool/examples/postgres/persistence-debug-custom.json b/fhir-install-bulkdata/src/main/docker/fhir-schematool/examples/postgres/persistence-debug-custom.json new file mode 100644 index 00000000000..22404ffdeb3 --- /dev/null +++ b/fhir-install-bulkdata/src/main/docker/fhir-schematool/examples/postgres/persistence-debug-custom.json @@ -0,0 +1,8 @@ +{ + "persistence": [ + { + "behavior": "custom", + "sql": "--create-schemas" + } + ] +} \ No newline at end of file diff --git a/fhir-install-bulkdata/src/main/docker/fhir-schematool/examples/postgres/persistence-debug-example.json b/fhir-install-bulkdata/src/main/docker/fhir-schematool/examples/postgres/persistence-debug-example.json new file mode 100644 index 00000000000..01baa83cf61 --- /dev/null +++ b/fhir-install-bulkdata/src/main/docker/fhir-schematool/examples/postgres/persistence-debug-example.json @@ -0,0 +1,7 @@ +{ + "persistence": [ + { + "behavior": "debug" + } + ] +} \ No newline at end of file diff --git a/fhir-install-bulkdata/src/main/docker/fhir-schematool/examples/postgres/persistence-offboard-example.json b/fhir-install-bulkdata/src/main/docker/fhir-schematool/examples/postgres/persistence-offboard-example.json new file mode 100644 index 00000000000..4aced384ba9 --- /dev/null +++ b/fhir-install-bulkdata/src/main/docker/fhir-schematool/examples/postgres/persistence-offboard-example.json @@ -0,0 +1,27 @@ +{ + "persistence": [ + { + "db": { + "host": "172.17.0.3", + "port": "50000", + "database": "fhirdb", + "user": "postgres", + "password": "change-password", + "type": "postgresql", + "ssl": "false", + "certificate_base64": "empty" + }, + "tenant": { + "name": "default3", + "key": "custom-key-here" + }, + "schema": { + "fhir": "fhirdata", + "batch": "", + "oauth": "" + }, + "grant": "fhirserver", + "behavior": "offboard" + } + ] +} \ No newline at end of file diff --git a/fhir-install-bulkdata/src/main/docker/fhir-schematool/examples/postgres/persistence-onboard-example.json b/fhir-install-bulkdata/src/main/docker/fhir-schematool/examples/postgres/persistence-onboard-example.json new file mode 100644 index 00000000000..b9ec6e5aa8b --- /dev/null +++ b/fhir-install-bulkdata/src/main/docker/fhir-schematool/examples/postgres/persistence-onboard-example.json @@ -0,0 +1,27 @@ +{ + "persistence": [ + { + "db": { + "host": "172.17.0.3", + "port": "5432", + "database": "fhirdb", + "user": "postgres", + "password": "change-password", + "type": "postgresql", + "ssl": "false", + "certificate_base64": "empty" + }, + "tenant": { + "name": "default3", + "key": "custom-key-here" + }, + "schema": { + "fhir": "fhirdata", + "batch": "", + "oauth": "" + }, + "grant": "fhirserver", + "behavior": "onboard" + } + ] +} \ No newline at end of file diff --git a/fhir-install-bulkdata/src/main/docker/fhir-schematool/run.sh b/fhir-install-bulkdata/src/main/docker/fhir-schematool/run.sh new file mode 100644 index 00000000000..0bfdd23e4d1 --- /dev/null +++ b/fhir-install-bulkdata/src/main/docker/fhir-schematool/run.sh @@ -0,0 +1,510 @@ +#!/usr/bin/env bash + +# ---------------------------------------------------------------------------- +# (C) Copyright IBM Corp. 2022 +# +# SPDX-License-Identifier: Apache-2.0 +# ---------------------------------------------------------------------------- + +set -e -o pipefail + +# When debugging -x is an accepted practice, when not debugging, please comment out. +# set -x + +############################################################################## +# The global variables used are: + +SCRIPT_NAME="$(basename ${BASH_SOURCE[0]})" + +# Override this when testing locally. +SCHEMA_TOOL_LOCATION="/opt/schematool" + +# Creates the temporary TOOL_INPUT lifecycle is the running script. +TOOL_INPUT_FILE=$(mktemp) || { echo "Failed to create the temporary properties file"; exit 1; } +TOOL_INPUT_USED="${ENV_TOOL_INPUT}" + +# Tool Input - Acceptable is console or file +TOOL_OUTPUT_USED=${ENV_TOOL_OUTPUT} +[ -z "${TOOL_OUTPUT_USED}" ] && TOOL_OUTPUT_USED="file" +TOOL_OUTPUT_FILE="/opt/schematool/output.`date +%F_%H-%M-%S`" + +# Debug - Flags if DEBUG is true or anything else +TOOL_DEBUG=${ENV_TOOL_DEBUG} +[ -z "${TOOL_DEBUG}" ] && TOOL_DEBUG="false" + +# SKIP - initial value is forced to lowercase +TOOL_SKIP=`echo "${ENV_SKIP}" | tr '[:upper:]' '[:lower:]'` +[ -z "${TOOL_SKIP}" ] && TOOL_SKIP="false" + +# Arguments: +TOOL_ARGS="$@" + +############################################################################## +# Helper Functions + +# debug - labels messages as debug +function debug { + if [ "$TOOL_DEBUG" = "true" ] + then + echo "${SCRIPT_NAME} - [DEBUG]: $(date +"%Y-%m-%d_%T") - ${1}" + fi +} + +# info - labels messages as info +function info { + echo "${SCRIPT_NAME} - [INFO]: $(date +"%Y-%m-%d_%T") - ${1}" +} + +# error_warn - labels messages as error +function error_warn { + echo "${SCRIPT_NAME} - [ERROR]: $(date +"%Y-%m-%d_%T") - ${1}" +} + +# get_property - gets the property value from the input_file +# returns (echo) the property from the INPUT_FILE +function get_property { + PROP_NAME="${1}" + JQ="${2}" + PROP_VALUE=`cat ${TOOL_INPUT_FILE} | grep ${PROP_NAME}= | sed 's|=| |' | awk '{print $2}'` + if [ ! -z "${PROP_VALUE}" ] + then + echo ${PROP_VALUE} + else + # only run if the file exists + if [ -f /opt/schematool/workarea/persistence.json ] + then + PROP_VALUE=$(/opt/schematool/jq -r ${JQ} /opt/schematool/workarea/persistence.json) + echo ${PROP_VALUE} + fi + fi +} + +############################################################################## +# Environment Processing Functions + +# process_cmd_properties - takes the environment properties +# and serializes to a temporary file in the container, which is destroyed at the end of the +# schell script. +function process_cmd_properties { + if [ ! -z "${TOOL_INPUT_USED}" ] + then + OS_TYPE="$(uname -s)" + case "${OS_TYPE}" in + Linux*) + # Since the pipe is used in a couple places where it can fail + # we don't want it to kill this script, and have overriden (temporarily) + # the fail on pipe and errexit, we'll control the exits. + set +o errexit + set +o pipefail + # only run if the file doesn't exist (we default to the existing file as it's been mounted) + if [ ! -f /opt/schematool/workarea/persistence.json ] + then + # originally there was error handling following this to check if the contents are valid json. + # We have opted to verify inline to each call. + echo -n "${TOOL_INPUT_USED}" | base64 -d > /opt/schematool/workarea/persistence.json 2> /dev/null || true + # This is to check if we have possible plain text. + RC=$(cat /opt/schematool/workarea/persistence.json | wc -l ) + if [ "${RC}" = "0" ] + then + echo "${TOOL_INPUT_USED}" | /opt/schematool/jq -r '.' > /opt/schematool/workarea/persistence.json + fi + fi + set -o errexit + set -o pipefail + ;; + Darwin*) + # Since the pipe is used in a couple places where it can fail + # we don't want it to kill this script, and have overriden (temporarily) + # the fail on pipe and errexit, we'll control the exits. + set +o errexit + set +o pipefail + # only run if the file doesn't exist (we default to the existing file as it's been mounted) + if [ ! -f /opt/schematool/workarea/persistence.json ] + then + echo -n "${TOOL_INPUT_USED}" | base64 --decode > /opt/schematool/workarea/persistence.json 2> /dev/null || true + # This is to check if we have possible plain text. + RC=$(cat /opt/schematool/workarea/persistence.json | wc -l ) + if [ "${RC}" = "0" ] + then + echo "${TOOL_INPUT_USED}" | /opt/schematool/jq -r '.' > /opt/schematool/workarea/persistence.json + fi + fi + set -o errexit + set -o pipefail + ;; + *) + exit 1 + ;; + esac + fi + + # Take the commandline and dump it out to the temporary inputfile. + echo -n "" > ${TOOL_INPUT_FILE} + for TOOL_ARG in ${TOOL_ARGS} + do + echo "${TOOL_ARG}" | sed 's|--||g' >> ${TOOL_INPUT_FILE} + done +} + +# process_cert - extracts the cert from the JSON file +# This is only valid if it's postgres +function process_cert { + DB_CERT=$(get_property db.cert .persistence[0].db.certificate_base64) + if [ ! -z "${DB_CERT}" ] && [ $DB_CERT != 'empty' ] + then + echo ${DB_CERT} | base64 -d > /opt/schematool/workarea/db.cert + fi +} + +############################################################################## +# Database Specific Wrapper for Commmon Calls + +# _call_postgres - local function to call postgres +function _call_postgres { + INPUT="$1" + + # Get the variables + DB_HOSTNAME=$(get_property db.host .persistence[0].db.host) + DB_PORT=$(get_property db.port .persistence[0].db.port) + DB_NAME=$(get_property db.database .persistence[0].db.database) + DB_USER=$(get_property user .persistence[0].db.user) + DB_PASSWORD=$(get_property password .persistence[0].db.password) + + # Check the SSL config and create the SSL_STANZA + DB_SSL_PG=$(get_property ssl .persistence[0].db.ssl) + SSL_STANZA="" + if [ "${DB_SSL_PG}" = "true" ] + then + SSL_STANZA="--prop ssl=true --prop sslmode=verify-full --prop sslrootcert=/opt/schematool/workarea/db.cert " + fi + + # since we are generating, we can debug this... with set +x + set -x + /opt/java/openjdk/bin/java -Dlog.dir=${SCHEMA_TOOL_LOCATION}/workarea -jar ${SCHEMA_TOOL_LOCATION}/fhir-persistence-schema-*-cli.jar \ + --prop "db.host=${DB_HOSTNAME}" \ + --prop "db.port=${DB_PORT}" \ + --prop "db.database=${DB_NAME}" \ + --prop "user=${DB_USER}" \ + --prop "password=${DB_PASSWORD}" \ + ${SSL_STANZA} \ + --db-type postgresql \ + ${INPUT} 2>&1 | tee ${SCHEMA_TOOL_LOCATION}/workarea/out.log + set +x +} + +############################################################################## +# General Database Functions + +# grant_to_dbuser - grant dbuser so the IBM FHIR Server can use it. +function grant_to_dbuser { + DB_TYPE=$(get_property db.type .persistence[0].db.type) + + # Get the tenant variable + TARGET_USER=$(get_property grant.to .persistence[0].grant) + # null is set here when the value doesn't actually exist. + if [ -z "${TARGET_USER}" ] || [ "null" = "${TARGET_USER}" ] + then + error_warn "Target User is not set and we are skipping the grant phase, it is recommended to run" + else + # Pick off the schemas + SCHEMA_OAUTH=$(get_property schema.name.oauth .persistence[0].schema.oauth) + if [ -z "${SCHEMA_OAUTH}" ] || [ "null" = "${SCHEMA_OAUTH}" ] + then + SCHEMA_OAUTH="FHIR_OAUTH" + fi + SCHEMA_BATCH=$(get_property schema.name.batch .persistence[0].schema.batch) + if [ -z "${SCHEMA_BATCH}" ] || [ "null" = "${SCHEMA_BATCH}" ] + then + SCHEMA_BATCH="FHIR_JBATCH" + fi + SCHEMA_FHIR=$(get_property schema.name.fhir .persistence[0].schema.fhir) + if [ -z "${SCHEMA_FHIR}" ] || [ "null" = "${SCHEMA_FHIR}" ] + then + SCHEMA_FHIR="FHIRDATA" + fi + + if [ "${DB_TYPE}" = "postgresql" ] + then + _call_postgres "--grant-to ${TARGET_USER} --target BATCH ${SCHEMA_BATCH} --target OAUTH "${SCHEMA_OAUTH}" --target DATA ${SCHEMA_FHIR} --pool-size 5" + fi + fi +} + +# drop_schema - drops the schema +# SCHEMA - the command to drop the schema +function drop_schema { + SCHEMA=$1 + DB_TYPE=$(get_property db.type .persistence[0].db.type) + if [ "${DB_TYPE}" = "postgresql" ] + then + # Pick off the schemas + SCHEMA_OAUTH=$(get_property schema.name.oauth .persistence[0].schema.oauth) + if [ -z "${SCHEMA_OAUTH}" ] || [ "null" = "${SCHEMA_OAUTH}" ] + then + SCHEMA_OAUTH="FHIR_OAUTH" + fi + SCHEMA_BATCH=$(get_property schema.name.batch .persistence[0].schema.batch) + if [ -z "${SCHEMA_BATCH}" ] || [ "null" = "${SCHEMA_BATCH}" ] + then + SCHEMA_BATCH="FHIR_JBATCH" + fi + SCHEMA_FHIR=$(get_property schema.name.fhir .persistence[0].schema.fhir) + if [ -z "${SCHEMA_FHIR}" ] || [ "null" = "${SCHEMA_FHIR}" ] + then + SCHEMA_FHIR="FHIRDATA" + fi + _call_postgres "--drop-schema-fhir --schema-name ${SCHEMA_FHIR} --drop-schema-batch ${SCHEMA_BATCH} --drop-schema-oauth ${SCHEMA_OAUTH} --pool-size 2 --confirm-drop --drop-admin" + fi +} + +# drop_schema_fhir - calls the drop_schema for a specific schema +function drop_schema_fhir { + DB_TYPE=$(get_property db.type .persistence[0].db.type) + MORE_TENANTS=$(has_more_tenants) + + SCHEMA_FHIR=$(get_property schema.name.fhir .persistence[0].schema.fhir) + if [ -z "${MORE_TENANTS}" ] + then + if [ -z "${SCHEMA_FHIR}" ] || [ "null" = "${SCHEMA_FHIR}" ] + then + SCHEMA_FHIR="FHIRDATA" + fi + drop_schema "--drop-schema-fhir --schema-name ${SCHEMA_FHIR}" + fi +} + +# drop_schema_oauth - calls the drop_schema for a specific schema +function drop_schema_oauth { + DB_TYPE=$(get_property db.type .persistence[0].db.type) + MORE_TENANTS=$(has_more_tenants) + + DB_SCHEMA=$(get_property schema.name.oauth .persistence[0].schema.oauth) + if [ -z "${MORE_TENANTS}" ] + then + if [ -z "${DB_SCHEMA}" ] || [ "null" = "${DB_SCHEMA}" ] + then + DB_SCHEMA="FHIR_OAUTH" + fi + drop_schema "--drop-schema-oauth --schema-name ${DB_SCHEMA}" + fi +} + +# drop_schema_batch - calls the drop_schema for a specific schema +function drop_schema_batch { + DB_TYPE=$(get_property db.type .persistence[0].db.type) + MORE_TENANTS=$(has_more_tenants) + + DB_SCHEMA=$(get_property schema.name.batch .persistence[0].schema.batch) + if [ -z "${MORE_TENANTS}" ] + then + if [ -z "${DB_SCHEMA}" ] || [ "null" = "${DB_SCHEMA}" ] + then + DB_SCHEMA="FHIR_JBATCH" + fi + drop_schema "--drop-schema-fhir --schema-name ${DB_SCHEMA}" + fi +} + +# create_schema - creates the schemas +# reverts to the default schemas when it is not set. +function create_schema { + info "creating the schema" + DB_TYPE=$(get_property db.type .persistence[0].db.type) + if [ "${DB_TYPE}" = "postgresql" ] + then + # Pick off the schemas + SCHEMA_OAUTH=$(get_property schema.name.oauth .persistence[0].schema.oauth) + if [ -z "${SCHEMA_OAUTH}" ] || [ "null" = "${SCHEMA_OAUTH}" ] + then + SCHEMA_OAUTH="FHIR_OAUTH" + fi + SCHEMA_BATCH=$(get_property schema.name.batch .persistence[0].schema.batch) + if [ -z "${SCHEMA_BATCH}" ] || [ "null" = "${SCHEMA_BATCH}" ] + then + SCHEMA_BATCH="FHIR_JBATCH" + fi + SCHEMA_FHIR=$(get_property schema.name.fhir .persistence[0].schema.fhir) + if [ -z "${SCHEMA_FHIR}" ] || [ "null" = "${SCHEMA_FHIR}" ] + then + SCHEMA_FHIR="FHIRDATA" + fi + _call_postgres "--create-schema-oauth ${SCHEMA_OAUTH} --create-schema-batch ${SCHEMA_BATCH} --create-schema-fhir ${SCHEMA_FHIR} --pool-size 2" + fi + info "done creating the schema" +} + +# update_schema +function update_schema { + DB_TYPE=$(get_property db.type .persistence[0].db.type) + if [ "${DB_TYPE}" = "postgresql" ] + then + SCHEMA_OAUTH=$(get_property schema.name.oauth .persistence[0].schema.oauth) + if [ -z "${SCHEMA_OAUTH}" ] || [ "null" = "${SCHEMA_OAUTH}" ] + then + SCHEMA_OAUTH="FHIR_OAUTH" + fi + SCHEMA_BATCH=$(get_property schema.name.batch .persistence[0].schema.batch) + if [ -z "${SCHEMA_BATCH}" ] || [ "null" = "${SCHEMA_BATCH}" ] + then + SCHEMA_BATCH="FHIR_JBATCH" + fi + SCHEMA_FHIR=$(get_property schema.name.fhir .persistence[0].schema.fhir) + if [ -z "${SCHEMA_FHIR}" ] || [ "null" = "${SCHEMA_FHIR}" ] + then + SCHEMA_FHIR="FHIRDATA" + fi + _call_postgres "--update-schema-fhir ${SCHEMA_FHIR} --update-schema-batch ${SCHEMA_BATCH} --update-schema-oauth ${SCHEMA_OAUTH} --pool-size 1" + fi +} + +# create_database_configuration_file - create the database configuration +function create_database_configuration_file { + process_cmd_properties $@ + process_cert +} + +# check_connectivity - checks the following connectivity: +# 1 - network CONNECTED +# 2 - Database Credentials +function check_connectivity { + check_network_path + check_database_credentials +} + +# check_network_path - checks the network path, if this fails a network path is not open. +function check_network_path { + DB_HOSTNAME=$(get_property db.host .persistence[0].db.host) + DB_PORT=$(get_property db.port .persistence[0].db.port) + DB_SSL_PG=$(get_property ssl .persistence[0].db.ssl) + + # intentionally using ftp as it's generally available, and make no difference on this check + info "Activating Network Path Connection..." + + # Since the pipe is used in a couple places where it can fail + # we don't want it to kill this script, and have overriden (temporarily) + # the fail on pipe and errexit, we'll control the exits. + set +o errexit + set +o pipefail + CONNECTED=$(echo '' | nc ${DB_HOSTNAME} ${DB_PORT} -v 2>&1) + if [ "$(echo $CONNECTED | grep -c Connected)" = "0" ] + then + info "[NETWORK_PATH_CHECK] - FAILURE - Unable to connect to database - ${DB_HOSTNAME}:${DB_PORT}" + exit 5; + else + info "[NETWORK_PATH_CHECK] - SUCCESS - Connected to the database - ${DB_HOSTNAME}:${DB_PORT}" + fi + set -o errexit + set -o pipefail +} + +# check_database_credentials - checks the database credentials using the schema tool +function check_database_credentials { + DB_TYPE=$(get_property db.type .persistence[0].db.type) + if [ "${DB_TYPE}" = "postgresql" ] + then + DB_HOSTNAME=$(get_property db.host .persistence[0].db.host) + DB_PORT=$(get_property db.port .persistence[0].db.port) + DB_USER=$(get_property user .persistence[0].db.user) + DB_PASSWORD=$(get_property password .persistence[0].db.password) + DB_NAME=$(get_property db.database .persistence[0].db.database) + + DB_SSL_PG=$(get_property ssl .persistence[0].db.ssl) + SSL_CONNECTION="false" + if [ "$DB_SSL_PG" = "true" ] + then + SSL_CONNECTION="true" + echo "${SSL_CONNECTION} is set" + fi + + # TODO: implement a nice little check here that verifies the user,password can connect. + debug "postgres doesn't have as easy a tool to use" + fi +} + +# onboard_behavior - creates and updates a schema and apply grants +function onboard_behavior { + if [ "${TOOL_SKIP}" != "true" ] + then + info "Starting the onboard behavior" + check_connectivity + create_schema + update_schema + grant_to_dbuser + else + info "The onboarding deployment flow is skipped" + fi +} + +# offboard_behavior - Removes data when there are no tenants left. +function offboard_behavior { + if [ "${TOOL_SKIP}" != "true" ] + then + info "Starting the offboard behavior" + + # All Databases + drop_schema_fhir + drop_schema_batch + drop_schema_oauth + else + info "The offboarding deployment flow is skipped" + fi +} + +# custom_behavior - runs a custom action +function custom_behavior { + if [ "${TOOL_SKIP}" != "true" ] + then + info "Starting the custom behavior" + + if [ "${DB_TYPE}" = "postgresql" ] + then + _call_postgres "$@" + fi + else + info "The custom behavior flow is skipped" + fi +} + +# debug_behavior - runs the debug action +function debug_behavior { + info "The files included with the tool are:" + ls -alh /opt/schematool + echo "The OpenSSL version is:" + openssl version +} + +# process_behavior - translates the identified behavior to lowercase +# the behavior is picked up from the existing variable +# +# @implNote each behavior that the tool is expected to have must have a case statement. +function process_behavior { + TOOL_BEHAVIOR=$(get_property tool.behavior .persistence[0].behavior | tr '[:upper:]' '[:lower:]') + echo "The tool behavior being executed is $TOOL_BEHAVIOR" + case $TOOL_BEHAVIOR in + onboard) + onboard_behavior + ;; + offboard) + offboard_behavior + ;; + custom) + custom_behavior "$@" + ;; + debug) + debug_behavior + ;; + *) + info "invalid behavior called, dropping through - '${TOOL_BEHAVIOR}'" + ;; + esac +} + +############################################################################## +# The logic is activated in the following calls: + +debug "Script Name is ${SCRIPT_NAME}" + +create_database_configuration_file "$@" +process_behavior "$@" + +# EOF diff --git a/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/1795/1795.sh b/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/1795/1795.sh new file mode 100644 index 00000000000..4f88e77f6bb --- /dev/null +++ b/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/1795/1795.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +# ---------------------------------------------------------------------------- +# (C) Copyright IBM Corp. 2022 +# +# SPDX-License-Identifier: Apache-2.0 +# ---------------------------------------------------------------------------- + +# Issue - https://github.com/LinuxForHealth/FHIR/issues/1795 +# Create Schemas should support in multiple ways. + +# Pre Condition: +# 1 - linuxforhealth/fhir-schematool must be built based on the latest. +# docker build -t linuxforhealth/fhir-schematool:latest . + +# Post Condition: +# 1 - Postgres should be started +# 2 - Tool should complete without any issues/exceptions +# 3 - The following schemas should exist +# fhir_admin +# fhirdata3 +# fhiroauth3 +# fhirbatch3 +# + +docker-compose up -d db + +# schemas that end with 3 +docker-compose run tool --tool.behavior=onboard --db.type=postgresql \ + --db.host=db --db.port=5432 --db.database=fhirdb --schema.name.fhir=fhirdata3 \ + --schema.name.batch=fhirbatch3 --schema.name.oauth=fhiroauth3 \ + --user=postgres --password=change-password + +# schemas that end with 43, except for oauth as default +docker-compose run tool --tool.behavior=onboard --db.type=postgresql \ + --db.host=db --db.port=5432 --db.database=fhirdb --schema.name.fhir=fhirdata43 \ + --schema.name.batch=fhirbatch43 \ + --user=postgres --password=change-password + +# schemas that uses all defaults +docker-compose run tool --tool.behavior=onboard --db.type=postgresql \ + --db.host=db --db.port=5432 --db.database=fhirdb \ + --user=postgres --password=change-password + +# # schemas that end with 50 for fhirdata only +docker-compose run tool --tool.behavior=onboard --db.type=postgresql \ + --db.host=db --db.port=5432 --db.database=fhirdb --schema.name.fhir=fhirdata50 \ + --user=postgres --password=change-password + +echo SELECT schema_name FROM information_schema.schemata | docker-compose exec -T -e PGPASSWORD=change-password db psql -h db -U postgres fhirdb + +docker-compose down -t 1 +# EOF \ No newline at end of file diff --git a/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/1795/docker-compose.yml b/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/1795/docker-compose.yml new file mode 100644 index 00000000000..6d7a4e287aa --- /dev/null +++ b/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/1795/docker-compose.yml @@ -0,0 +1,44 @@ +version: '3.7' +services: + db: + image: postgres:12-alpine + restart: always + environment: + POSTGRES_PASSWORD: change-password + POSTGRES_USER: postgres + POSTGRES_DB: fhirdb + PGDATA: /data/postgres + shm_size: 256MB + tty: true + stdin_open: true + hostname: db + sysctls: + net.core.somaxconn: 256 + net.ipv4.tcp_syncookies: 0 + privileged: true + networks: + - fhir + ports: + - "5432:5432" + tool: + image: linuxforhealth/fhir-schematool:latest + restart: always + environment: + PLACEHOLDER: change-password + tty: true + stdin_open: true + hostname: tool + stop_grace_period: 2m + sysctls: + net.core.somaxconn: 256 + net.ipv4.tcp_syncookies: 0 + # To use all of 2 gigabytes + kernel.shmmax: 1055092736 + kernel.shmall: 257591 + privileged: true + command: tail -f /dev/null + networks: + - fhir +networks: + fhir: + driver: bridge \ No newline at end of file diff --git a/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/1796/1796.sh b/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/1796/1796.sh new file mode 100644 index 00000000000..ba761790d7c --- /dev/null +++ b/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/1796/1796.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# ---------------------------------------------------------------------------- +# (C) Copyright IBM Corp. 2022 +# +# SPDX-License-Identifier: Apache-2.0 +# ---------------------------------------------------------------------------- + +# Issue - https://github.com/LinuxForHealth/FHIR/issues/1796 + +# Don't log out the file isn't found warning. + +# Pre Condition: +# 1 - linuxforhealth/fhir-schematool must be built based on the latest. +# docker build -t linuxforhealth/fhir-schematool:latest . + +# Post Condition: +# 1 - Postgres should be started +# 2 - No Error Message like 'jq: error: Could not open file /opt/schematool/workarea/persistence.json' + +docker-compose up -d db + +# Startup +docker-compose run tool --tool.behavior=onboard --db.type=postgresql \ + --db.host=db --db.port=5432 --db.database=fhirdb --schema.name.fhir=fhirdata3 \ + --user=postgres --password=change-password + +docker-compose down -t 1 +# EOF \ No newline at end of file diff --git a/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/1796/docker-compose.yml b/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/1796/docker-compose.yml new file mode 100644 index 00000000000..6d7a4e287aa --- /dev/null +++ b/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/1796/docker-compose.yml @@ -0,0 +1,44 @@ +version: '3.7' +services: + db: + image: postgres:12-alpine + restart: always + environment: + POSTGRES_PASSWORD: change-password + POSTGRES_USER: postgres + POSTGRES_DB: fhirdb + PGDATA: /data/postgres + shm_size: 256MB + tty: true + stdin_open: true + hostname: db + sysctls: + net.core.somaxconn: 256 + net.ipv4.tcp_syncookies: 0 + privileged: true + networks: + - fhir + ports: + - "5432:5432" + tool: + image: linuxforhealth/fhir-schematool:latest + restart: always + environment: + PLACEHOLDER: change-password + tty: true + stdin_open: true + hostname: tool + stop_grace_period: 2m + sysctls: + net.core.somaxconn: 256 + net.ipv4.tcp_syncookies: 0 + # To use all of 2 gigabytes + kernel.shmmax: 1055092736 + kernel.shmall: 257591 + privileged: true + command: tail -f /dev/null + networks: + - fhir +networks: + fhir: + driver: bridge \ No newline at end of file diff --git a/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/1797/1797.sh b/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/1797/1797.sh new file mode 100644 index 00000000000..87fea6ed73a --- /dev/null +++ b/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/1797/1797.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# ---------------------------------------------------------------------------- +# (C) Copyright IBM Corp. 2022 +# +# SPDX-License-Identifier: Apache-2.0 +# ---------------------------------------------------------------------------- + +# Issue - https://github.com/LinuxForHealth/FHIR/issues/1797 + +# Don't log out the --pool-size + +# Pre Condition: +# 1 - linuxforhealth/fhir-schematool must be built based on the latest. +# docker build -t linuxforhealth/fhir-schematool:latest . + +# Post Condition: +# 1 - Postgres should be started +# 2 - No Error Message like 'java.lang.IllegalArgumentException: Invalid SQL object name: --pool-size' + +docker-compose up -d db + +# Startup +docker-compose run tool --tool.behavior=onboard --db.type=postgresql \ + --db.host=db --db.port=5432 --db.database=fhirdb --schema.name.fhir=fhirdata \ + --user=postgres --password=change-password + +# Check the log + +docker-compose down -t 1 +# EOF \ No newline at end of file diff --git a/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/1797/docker-compose.yml b/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/1797/docker-compose.yml new file mode 100644 index 00000000000..6d7a4e287aa --- /dev/null +++ b/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/1797/docker-compose.yml @@ -0,0 +1,44 @@ +version: '3.7' +services: + db: + image: postgres:12-alpine + restart: always + environment: + POSTGRES_PASSWORD: change-password + POSTGRES_USER: postgres + POSTGRES_DB: fhirdb + PGDATA: /data/postgres + shm_size: 256MB + tty: true + stdin_open: true + hostname: db + sysctls: + net.core.somaxconn: 256 + net.ipv4.tcp_syncookies: 0 + privileged: true + networks: + - fhir + ports: + - "5432:5432" + tool: + image: linuxforhealth/fhir-schematool:latest + restart: always + environment: + PLACEHOLDER: change-password + tty: true + stdin_open: true + hostname: tool + stop_grace_period: 2m + sysctls: + net.core.somaxconn: 256 + net.ipv4.tcp_syncookies: 0 + # To use all of 2 gigabytes + kernel.shmmax: 1055092736 + kernel.shmall: 257591 + privileged: true + command: tail -f /dev/null + networks: + - fhir +networks: + fhir: + driver: bridge \ No newline at end of file diff --git a/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/1802/1802-base64.sh b/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/1802/1802-base64.sh new file mode 100644 index 00000000000..f34eef4f7e0 --- /dev/null +++ b/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/1802/1802-base64.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +# ---------------------------------------------------------------------------- +# (C) Copyright IBM Corp. 2022 +# +# SPDX-License-Identifier: Apache-2.0 +# ---------------------------------------------------------------------------- + +# Issue - https://github.com/LinuxForHealth/FHIR/issues/1802 + +# Pre Condition: +# 1 - linuxforhealth/fhir-schematool must be built based on the latest. +# docker build -t linuxforhealth/fhir-schematool:latest . + +# Post Condition: +# 1 - Postgres should be started +# 2 - No Errors and the schema should work. +# 3 - Check Version History + +# 1 - Start the db +docker-compose up -d db + +# 2 - Startup +docker-compose run -e ENV_TOOL_INPUT=$(cat 1802-onboarding.json | base64) tool + +# The schemas should exist +echo SELECT schema_name FROM information_schema.schemata | docker-compose exec -T -e PGPASSWORD=change-password db psql -h db -U postgres fhirdb + +# Should be 2700+ +echo 'SELECT COUNT(*) FROM FHIR_ADMIN.VERSION_HISTORY' | docker-compose exec -T -e PGPASSWORD=change-password db psql -h db -U postgres fhirdb + +docker-compose down -t 1 +# EOF \ No newline at end of file diff --git a/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/1802/1802-onboarding.json b/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/1802/1802-onboarding.json new file mode 100644 index 00000000000..3a4ca20c92d --- /dev/null +++ b/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/1802/1802-onboarding.json @@ -0,0 +1,22 @@ +{ + "persistence": [ + { + "db": { + "host": "db", + "port": "5432", + "database": "fhirdb", + "user": "postgres", + "password": "change-password", + "type": "postgresql", + "ssl": "false", + "certificate_base64": "empty" + }, + "schema": { + "fhir": "fhirdata", + "batch": "", + "oauth": "" + }, + "behavior": "onboard" + } + ] +} \ No newline at end of file diff --git a/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/1802/1802-plain.sh b/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/1802/1802-plain.sh new file mode 100644 index 00000000000..c348da737ce --- /dev/null +++ b/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/1802/1802-plain.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +# ---------------------------------------------------------------------------- +# (C) Copyright IBM Corp. 2022 +# +# SPDX-License-Identifier: Apache-2.0 +# ---------------------------------------------------------------------------- + +# Issue - https://github.com/LinuxForHealth/FHIR/issues/1802 + +# Pre Condition: +# 1 - linuxforhealth/fhir-schematool must be built based on the latest. +# docker build -t linuxforhealth/fhir-schematool:latest . + +# Post Condition: +# 1 - Postgres should be started +# 2 - No Errors and the schema should work. +# 3 - Check Version History + +# 1 - Start the db +docker-compose up -d db + +# 2 - Startup +docker-compose run -e ENV_TOOL_INPUT="$(cat 1802-onboarding.json)" tool + +# The schemas should exist +echo SELECT schema_name FROM information_schema.schemata | docker-compose exec -T -e PGPASSWORD=change-password db psql -h db -U postgres fhirdb + +# Should be 2700+ +echo 'SELECT COUNT(*) FROM FHIR_ADMIN.VERSION_HISTORY' | docker-compose exec -T -e PGPASSWORD=change-password db psql -h db -U postgres fhirdb + +docker-compose down -t 1 +# EOF \ No newline at end of file diff --git a/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/1802/docker-compose.yml b/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/1802/docker-compose.yml new file mode 100644 index 00000000000..353906f8e7e --- /dev/null +++ b/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/1802/docker-compose.yml @@ -0,0 +1,37 @@ +version: '3.7' +services: + db: + image: postgres:12-alpine + restart: always + environment: + POSTGRES_PASSWORD: change-password + POSTGRES_USER: postgres + POSTGRES_DB: fhirdb + PGDATA: /data/postgres + shm_size: 256MB + tty: true + stdin_open: true + hostname: db + sysctls: + net.core.somaxconn: 256 + net.ipv4.tcp_syncookies: 0 + privileged: true + ports: + - "5432:5432" + tool: + image: linuxforhealth/fhir-schematool:latest + restart: always + environment: + PLACEHOLDER: change-password + tty: true + stdin_open: true + hostname: tool + stop_grace_period: 2m + sysctls: + net.core.somaxconn: 256 + net.ipv4.tcp_syncookies: 0 + # To use all of 2 gigabytes + kernel.shmmax: 1055092736 + kernel.shmall: 257591 + privileged: true + command: tail -f /dev/null \ No newline at end of file diff --git a/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/cp/postgres-onboard-noschema.json b/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/cp/postgres-onboard-noschema.json new file mode 100644 index 00000000000..b9871a1a712 --- /dev/null +++ b/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/cp/postgres-onboard-noschema.json @@ -0,0 +1,22 @@ +{ + "persistence": [ + { + "db": { + "host": "**********.databases.appdomain.cloud", + "port": "32195", + "database": "ibmclouddb", + "user": "*****", + "password": "*****", + "type": "postgresql", + "ssl": "true", + "certificate_base64": "******=" + }, + "tenant": { + "name": "default", + "key": "custom-key-here" + }, + "grant": "fhirserver", + "behavior": "onboard" + } + ] +} \ No newline at end of file diff --git a/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/cp/postgres-onboard.json b/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/cp/postgres-onboard.json new file mode 100644 index 00000000000..3d925a7dbac --- /dev/null +++ b/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/cp/postgres-onboard.json @@ -0,0 +1,27 @@ +{ + "persistence": [ + { + "db": { + "host": "************.databases.appdomain.cloud", + "port": "32195", + "database": "ibmclouddb", + "user": "************", + "password": "************", + "type": "postgresql", + "ssl": "true", + "certificate_base64": "************" + }, + "tenant": { + "name": "default", + "key": "custom-key-here" + }, + "schema": { + "fhir": "fhirdata", + "batch": "", + "oauth": "" + }, + "grant": "fhirserver", + "behavior": "onboard" + } + ] +} \ No newline at end of file diff --git a/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/cp/test.sh b/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/cp/test.sh new file mode 100644 index 00000000000..5bc299d9d5b --- /dev/null +++ b/fhir-install-bulkdata/src/main/docker/fhir-schematool/tests/cp/test.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# ---------------------------------------------------------------------------- +# (C) Copyright IBM Corp. 2022 +# +# SPDX-License-Identifier: Apache-2.0 +# ---------------------------------------------------------------------------- + +docker run --env ENV_TOOL_INPUT=$(cat postgres-onboard.json | base64) \ + linuxforhealth/fhir-schematool:latest + +docker run --env ENV_TOOL_INPUT=$(cat postgres-onboard-noschema.json | base64) \ + linuxforhealth/fhir-schematool:latest diff --git a/fhir-install-bulkdata/src/main/java/.empty b/fhir-install-bulkdata/src/main/java/.empty new file mode 100644 index 00000000000..e69de29bb2d diff --git a/fhir-install-bulkdata/src/main/resources/docs/README.txt b/fhir-install-bulkdata/src/main/resources/docs/README.txt new file mode 100644 index 00000000000..d080d18c909 --- /dev/null +++ b/fhir-install-bulkdata/src/main/resources/docs/README.txt @@ -0,0 +1,5 @@ +The IBM FHIR Server User Guide has moved to https://linuxforhealth.github.io/FHIR/ + +The source is available at https://github.com/LinuxForHealth/fhir/tree/main/docs + +FHIR® is the registered trademark of HL7 and is used with the permission of HL7. diff --git a/fhir-install-bulkdata/src/main/resources/scripts/install.bat b/fhir-install-bulkdata/src/main/resources/scripts/install.bat new file mode 100644 index 00000000000..c96aafc87d9 --- /dev/null +++ b/fhir-install-bulkdata/src/main/resources/scripts/install.bat @@ -0,0 +1,138 @@ +@echo off +@REM ---------------------------------------------------------------------------- +@REM (C) Copyright IBM Corp. 2022 +@REM +@REM SPDX-License-Identifier: Apache-2.0 +@REM ---------------------------------------------------------------------------- + +SETLOCAL ENABLEDELAYEDEXPANSION + +set LIBERTY_VERSION=22.0.0.10 + +echo Executing %0 to deploy the fhir-bulkdata-server web application... + +@REM Make sure that JAVA_HOME is set +if "-%JAVA_HOME%-"=="--" ( + echo "Error: JAVA_HOME not set; make sure JAVA_HOME points to a Java 11 JVM and then re-try." + set rc=1 + goto :exit +) else ( + set JAVA_CMD="%JAVA_HOME%\bin\java.exe" + if not exist !JAVA_CMD! ( + echo Error: Incorrect JAVA_HOME value: %JAVA_HOME% + set rc=1 + goto :exit + ) +) + +@REM echo JAVA_HOME: %JAVA_HOME% +@REM echo JAVA_CMD: %JAVA_CMD% +@REM set rc=1 +@REM goto :exit + +@REM Determine the location of this script. +set BASEDIR=%~dp0 + +@REM Remove any trailing \ from BASEDIR +if %BASEDIR:~-1%==\ set BASEDIR=%BASEDIR:~0,-1% + +@REM Default liberty install location +cd %BASEDIR%\.. +set UNZIP_LOC=%CD% +set WLP_INSTALL_DIR=%UNZIP_LOC%\liberty-runtime + +@REM Allow user to override default install location +if not "-%1-"=="--" set WLP_INSTALL_DIR=%1 + +@REM Add a trailing \ to WLP_INSTALL_DIR if needed +if not "%WLP_INSTALL_DIR:~-1%"=="\" set WLP_INSTALL_DIR=%WLP_INSTALL_DIR%\ + +echo Deploying fhir-bulkdata-server in location: %WLP_INSTALL_DIR% + +@REM If the liberty install directory doesnt exist, then create it. +if not exist %WLP_INSTALL_DIR% ( + echo The Liberty installation directory does not exist; attempting to create it... + mkdir %WLP_INSTALL_DIR% + if errorlevel 1 ( + set rc=%ERRORLEVEL% + echo Error creating installation directory: %rc% + goto :exit + ) +) + +@REM Unzip liberty runtime zip +echo Extracting the Liberty runtime... +call :UnZip %BASEDIR%\openliberty-runtime-%LIBERTY_VERSION%.zip\ %WLP_INSTALL_DIR% +if %rc% neq 0 ( + echo Error extracting liberty runtime: %rc% + goto :exit +) + +@REM Save the liberty home directory. +set WLP_ROOT=%WLP_INSTALL_DIR%wlp + +@REM Create our server +echo Creating the Liberty defaultServer... +%COMSPEC% /c %WLP_ROOT%\bin\server.bat create defaultServer +if errorlevel 1 ( + set rc=%ERRORLEVEL% + echo Error creating server definition: %rc% + goto :exit +) + +@REM Copy our server assets +echo Deploying fhir-bulkdata-server assets to the server runtime environment. +xcopy /S /Y /Q %BASEDIR%\artifacts\* %WLP_ROOT%\usr\ +if errorlevel 1 ( + set rc=%ERRORLEVEL% + echo Error deploying fhir-bulkdata-server assets to server runtime environment: %rc% + goto :exit +) + + +echo The FHIR Bulkdata Server has been successfully deployed to the Liberty runtime +echo at: %WLP_ROOT% +echo The fhir-server application requires Java 11. +echo If you do not have one, a copy of the Java 11 SDK can be obtained at https://adoptium.net. +echo The following steps must be completed before the server can be started: +echo 1. Set the JAVA_HOME environment variable for your Java installation. +echo 2. If using a database other than the embedded derby one, make sure that your selected database +echo is active and ready to accept requests. +echo 3. Deploy the database schema via the fhir-persistence-schema cli jar under %BASEDIR%\tools +echo and grant necessary permissions. +echo 4. Modify the Liberty server config by adding/removing/modifying the XML snippets under +echo %WLP_ROOT%\usr\servers\defaultServer\configDropins to configure datasource definitions, +echo TLS configuration (keystores), webapp security, and more. +echo 5. Modify the FHIR Bulkdata server config (fhir-server-config.json) under +echo %WLP_ROOT%\usr\servers\defaultServer\config to configure the persistence, resource endpoints, +echo and related FHIR server features. +echo You can start and stop the server with these commands: +echo %WLP_ROOT%\bin\server.bat start +echo %WLP_ROOT%\bin\server.bat stop +set rc=0 +goto :exit + + +@REM This function will unzip %1 into the directory %2 +@REM by creating a VB script and executing it. +:UnZip +set vbs="%temp%\_.vbs" +if exist %vbs% del /f /q %vbs% +>%vbs% echo Set fso = CreateObject("Scripting.FileSystemObject") +>>%vbs% echo strDest = "%2" +>>%vbs% echo strZipFileName = "%1" +>>%vbs% echo If NOT fso.FolderExists(strDest) Then +>>%vbs% echo fso.CreateFolder(strDest) +>>%vbs% echo End If +>>%vbs% echo set objShell = CreateObject("Shell.Application") +>>%vbs% echo set FilesInZip=objShell.NameSpace(strZipFileName).items +>>%vbs% echo objShell.NameSpace(strDest).CopyHere(FilesInZip) +>>%vbs% echo Set fso = Nothing +>>%vbs% echo Set objShell = Nothing +cscript //nologo %vbs% +set rc=%ERRORLEVEL% +if exist %vbs% del /f /q %vbs% +goto :eof + +:exit +%COMSPEC% /c exit /b %rc% diff --git a/fhir-install-bulkdata/src/main/resources/scripts/install.sh b/fhir-install-bulkdata/src/main/resources/scripts/install.sh new file mode 100644 index 00000000000..4cb0ec4a5eb --- /dev/null +++ b/fhir-install-bulkdata/src/main/resources/scripts/install.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env sh +############################################################################### +# (C) Copyright IBM Corp. 2022 +# +# SPDX-License-Identifier: Apache-2.0 +############################################################################### + +export LIBERTY_VERSION="22.0.0.10" + +echo " +Executing $0 to deploy the fhir-bulkdata-server web application... +" + +# Determine the location of this script. +# basedir=`dirname "$0"` +cd $(dirname $0); basedir="$(pwd)" + +# Default liberty install location +UNZIP_LOC=`unset CDPATH; cd "$basedir/.." && pwd` +LIBERTY_INSTALL_DIR="${UNZIP_LOC}/liberty-runtime" + +# Allow user to override default install location +if [ $# -gt 0 ] +then + LIBERTY_INSTALL_DIR=$1 +fi + +echo "Deploying in location: ${LIBERTY_INSTALL_DIR}" + +# Save the liberty home directory. +LIBERTY_ROOT=${LIBERTY_INSTALL_DIR}/wlp + +if [ -d "$LIBERTY_ROOT" ]; then + # If the liberty install directory exists, make sure we actually have Liberty installed there. + if [ -f "${LIBERTY_ROOT}/bin/server" ]; then + echo "Using the existing Liberty installation:" + ${LIBERTY_ROOT}/bin/productInfo version + else + echo " +Invalid installation directory specified for Liberty runtime: ${LIBERTY_ROOT} +If the installation directory exists then it must contain a valid Liberty runtime." + exit 1 + fi +else + # If the liberty install directory doesn't exist, then create it. + echo "Extracting the Liberty runtime... " + unzip -qq ${basedir}/openliberty-runtime-${LIBERTY_VERSION}.zip -d ${LIBERTY_INSTALL_DIR} + rc=$? + if [ $rc != 0 ]; then + echo "Error extracting liberty runtime: $rc" + exit $rc + else + echo "done!" + fi +fi + +# Create the defaultServer if necessary. +if [ ! -d "${LIBERTY_ROOT}/usr/servers/defaultServer" ]; then + echo "Creating the Liberty defaultServer... " + ${LIBERTY_ROOT}/bin/server create defaultServer + rc=$? + if [ $rc != 0 ]; then + echo "Error creating server definition: $rc" + exit $rc + fi +fi + +# Copy our server assets +echo "Deploying fhir-bulkdata-server assets to the server runtime environment... " +cp -r ${basedir}/artifacts/* ${LIBERTY_ROOT}/usr/ +rc=$? +if [ $rc != 0 ]; then + echo "Error deploying fhir-bulkdata-server assets to server runtime environment: $rc" + exit $rc +else + echo "done!" +fi + +echo " + +The FHIR Bulkdata Server has been successfully deployed to the Liberty runtime +at: ${LIBERTY_ROOT} + +The fhir-bulkdata-server application requires Java 11. +If you do not have one, a copy of the Java 11 SDK can be obtained at https://adoptium.net. +The following steps must be completed before the server can be started: + +1) Set the JAVA_HOME environment variable for your Java installation. + +2) If using a database other than the embedded derby one, make sure that your selected database + is active and ready to accept requests. + +3) Deploy the database schema via the fhir-persistence-schema cli jar under ${basedir}/tools + and grant necessary permissions. + +4) Modify the Liberty server config (server.xml) by adding/removing/modifying the XML snippets under + ${LIBERTY_ROOT}/usr/servers/defaultServer/configDropins to configure datasource definitions, + TLS configuration (keystores), webapp security, and more. + +5) Modify the FHIR Bulkdata server config (fhir-server-config.json) under + ${LIBERTY_ROOT}/usr/servers/defaultServer/config to configure the persistence, resource endpoints, + and related FHIR server features. + +You can start and stop the server with these commands: + ${LIBERTY_ROOT}/bin/server start + ${LIBERTY_ROOT}/bin/server stop +" \ No newline at end of file diff --git a/fhir-install/pom.xml b/fhir-install/pom.xml index 9b9ff967840..b29c0f89c10 100644 --- a/fhir-install/pom.xml +++ b/fhir-install/pom.xml @@ -14,12 +14,6 @@ pom - - ${project.groupId} - fhir-bulkdata-webapp - ${project.version} - war - ${project.groupId} fhir-server-webapp diff --git a/fhir-install/src/main/assembly/distribution.xml b/fhir-install/src/main/assembly/distribution.xml index 027178adce9..61e5c525398 100644 --- a/fhir-install/src/main/assembly/distribution.xml +++ b/fhir-install/src/main/assembly/distribution.xml @@ -32,11 +32,6 @@ artifacts/servers/defaultServer/apps 0755 - - ../fhir-bulkdata-webapp/target/fhir-bulkdata-webapp.war - artifacts/servers/defaultServer/apps - 0755 - ../fhir-openapi/target/fhir-openapi.war artifacts/servers/defaultServer/apps diff --git a/fhir-install/src/main/docker/fhir-server/bootstrap.sh b/fhir-install/src/main/docker/fhir-server/bootstrap.sh index 022def62bd2..9c08036d947 100755 --- a/fhir-install/src/main/docker/fhir-server/bootstrap.sh +++ b/fhir-install/src/main/docker/fhir-server/bootstrap.sh @@ -50,11 +50,14 @@ function _call_derby_db { function _bootstrap_db { if [ "$PERFORM_BOOTSTRAP_DB" = "true" ] then + echo "Performing Derby database bootstrapping" info "Performing Derby database bootstrapping" _call_derby_db "--update-schema" + echo "Finished Derby database bootstrapping" info "Finished Derby database bootstrapping" else info "Skipping Derby database bootstrapping" + echo "Skipping Derby database bootstrapping" fi } diff --git a/fhir-parent/pom.xml b/fhir-parent/pom.xml index 760f9b32478..7f4f1f5f4e0 100644 --- a/fhir-parent/pom.xml +++ b/fhir-parent/pom.xml @@ -116,6 +116,7 @@ ../fhir-swagger-generator ../fhir-openapi ../fhir-install + ../fhir-install-bulkdata + + transportSecurity-1.0 + appSecurity-2.0 + jaxrs-2.1 + servlet-4.0 + jdbc-4.1 + websocket-1.1 + localConnector-1.0 + batch-1.0 + batchManagement-1.0 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/fhir-server-webapp/src/main/liberty/bulkdata/disabled/derby/datasource-bulkdata.xml b/fhir-server-webapp/src/main/liberty/bulkdata/disabled/derby/datasource-bulkdata.xml new file mode 100644 index 00000000000..969cf69ab31 --- /dev/null +++ b/fhir-server-webapp/src/main/liberty/bulkdata/disabled/derby/datasource-bulkdata.xml @@ -0,0 +1,7 @@ + + + + + + + diff --git a/fhir-server-webapp/src/main/liberty/bulkdata/disabled/postgres/datasource-bulkdata.xml b/fhir-server-webapp/src/main/liberty/bulkdata/disabled/postgres/datasource-bulkdata.xml new file mode 100644 index 00000000000..94712b9ec02 --- /dev/null +++ b/fhir-server-webapp/src/main/liberty/bulkdata/disabled/postgres/datasource-bulkdata.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + diff --git a/fhir-server-webapp/src/main/liberty/config/config/default/fhir-server-config.json b/fhir-server-webapp/src/main/liberty/config/config/default/fhir-server-config.json index 0a07a750ffe..8ca279c68b6 100644 --- a/fhir-server-webapp/src/main/liberty/config/config/default/fhir-server-config.json +++ b/fhir-server-webapp/src/main/liberty/config/config/default/fhir-server-config.json @@ -137,7 +137,7 @@ "enabled": true, "core": { "api": { - "url": "https://localhost:9443/ibm/api/batch", + "url": "https://localhost:8443/ibm/api/batch", "user": "fhiradmin", "password": "change-password", "truststore": "resources/security/fhirTrustStore.p12", diff --git a/fhir-server-webapp/src/main/liberty/config/configDropins/defaults/bulkdata.xml b/fhir-server-webapp/src/main/liberty/config/configDropins/defaults/bulkdata.xml deleted file mode 100644 index 59472cb8346..00000000000 --- a/fhir-server-webapp/src/main/liberty/config/configDropins/defaults/bulkdata.xml +++ /dev/null @@ -1,47 +0,0 @@ - - - - batch-1.0 - batchManagement-1.0 - jdbc-4.1 - servlet-4.0 - transportSecurity-1.0 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/fhir-server-webapp/src/main/liberty/config/configDropins/disabled/derby/bulkdata.xml b/fhir-server-webapp/src/main/liberty/config/configDropins/disabled/derby/bulkdata.xml deleted file mode 100644 index 587ed0e8a41..00000000000 --- a/fhir-server-webapp/src/main/liberty/config/configDropins/disabled/derby/bulkdata.xml +++ /dev/null @@ -1,43 +0,0 @@ - - - - batch-1.0 - batchManagement-1.0 - jdbc-4.1 - servlet-4.0 - transportSecurity-1.0 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/fhir-server-webapp/src/main/liberty/config/configDropins/disabled/postgres/bulkdata.xml b/fhir-server-webapp/src/main/liberty/config/configDropins/disabled/postgres/bulkdata.xml deleted file mode 100644 index 90cdc1cb0b7..00000000000 --- a/fhir-server-webapp/src/main/liberty/config/configDropins/disabled/postgres/bulkdata.xml +++ /dev/null @@ -1,55 +0,0 @@ - - - - batch-1.0 - batchManagement-1.0 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/fhir-server-webapp/src/test/liberty/config/config/default/fhir-server-config-postgresql-azurite.json b/fhir-server-webapp/src/test/liberty/config/config/default/fhir-server-config-postgresql-azurite.json index 22a03e4a643..921aae17f0d 100644 --- a/fhir-server-webapp/src/test/liberty/config/config/default/fhir-server-config-postgresql-azurite.json +++ b/fhir-server-webapp/src/test/liberty/config/config/default/fhir-server-config-postgresql-azurite.json @@ -321,7 +321,7 @@ "enabled": true, "core": { "api": { - "url": "https://localhost:9443/ibm/api/batch", + "url": "https://fhirbulkdata:9445/ibm/api/batch", "user": "fhiradmin", "password": "change-password", "truststore": "resources/security/fhirTrustStore.p12", diff --git a/fhir-server-webapp/src/test/liberty/config/config/default/fhir-server-config-postgresql-minio.json b/fhir-server-webapp/src/test/liberty/config/config/default/fhir-server-config-postgresql-minio.json index 149c5c8d8b7..5599f104390 100644 --- a/fhir-server-webapp/src/test/liberty/config/config/default/fhir-server-config-postgresql-minio.json +++ b/fhir-server-webapp/src/test/liberty/config/config/default/fhir-server-config-postgresql-minio.json @@ -311,7 +311,7 @@ "enabled": true, "core": { "api": { - "url": "https://localhost:9443/ibm/api/batch", + "url": "https://fhirbulkdata:9445/ibm/api/batch", "user": "fhiradmin", "password": "change-password", "truststore": "resources/security/fhirTrustStore.p12", diff --git a/operation/fhir-operation-bulkdata/src/main/java/org/linuxforhealth/fhir/operation/bulkdata/client/action/batch/BatchCancelRequestAction.java b/operation/fhir-operation-bulkdata/src/main/java/org/linuxforhealth/fhir/operation/bulkdata/client/action/batch/BatchCancelRequestAction.java index 5c8a1b6e068..c9edc54d89b 100644 --- a/operation/fhir-operation-bulkdata/src/main/java/org/linuxforhealth/fhir/operation/bulkdata/client/action/batch/BatchCancelRequestAction.java +++ b/operation/fhir-operation-bulkdata/src/main/java/org/linuxforhealth/fhir/operation/bulkdata/client/action/batch/BatchCancelRequestAction.java @@ -113,12 +113,7 @@ public void run(String tenant, String job) throws FHIROperationException { // Stop the Job's execution across all JobExecutions stopJobExecutions(jobExecutions); - - if (supportsDeleteJob()) { - deleteJob(job); - } else { - throw export.buildOperationException("Job deletion is not supported with the default Java Batch (in-memory) datastore. The Java Batch schema must be deployed and configured.", IssueType.NOT_SUPPORTED); - } + deleteJob(job); // Check for a server-side error if (Status.INTERNAL_SERVER_ERROR == result) { @@ -126,20 +121,6 @@ public void run(String tenant, String job) throws FHIROperationException { } } - /** - * @return True when there is a fhirbatchDb jndi instance, else, we have to assume this is in memory ONLY (also known as False). - */ - private boolean supportsDeleteJob() { - try { - Context ctx = new InitialContext(); - ctx.lookup("jdbc/fhirbatchDB"); - return true; - } catch (Exception ex) { - LOG.throwing(CLASSNAME, "supportsDeleteJob", ex); - return false; - } - } - /** * verify the tenant *