diff --git a/.env.cluster b/.env.cluster index b0f144a9..a1215e27 100644 --- a/.env.cluster +++ b/.env.cluster @@ -88,18 +88,18 @@ GF_SERVER_DOMAIN=grafana.domain KC_POSTGRES_REPLICA_SET=pgpool-1:5432,pgpool-2:5432,pgpool-3:5432 # Client Registry - JeMPI -JEMPI_WEB_INSTANCES=3 +JEMPI_WEB_INSTANCES=1 REACT_APP_JEMPI_BASE_API_HOST=https://jempi-api.domain REACT_APP_JEMPI_BASE_API_PORT=50000 JEMPI_SESSION_SECURE=true -JEMPI_REPMGR_PARTNER_NODES=jempi-postgresql-01,jempi-postgresql-02,jempi-postgresql-03 +JEMPI_REPMGR_PARTNER_NODES=jempi-postgresql-01 JEMPI_ASYNC_RECEIVER_INSTANCES=1 JEMPI_SYNC_RECEIVER_INSTANCES=1 JEMPI_PRE_PROCESSOR_INSTANCES=1 JEMPI_CONTROLLER_INSTANCES=1 JEMPI_EM_CALCULATOR_INSTANCES=1 JEMPI_LINKER_INSTANCES=1 -JEMPI_API_INSTANCES=3 +JEMPI_API_INSTANCES=1 # Resource limits OPENHIM_MEMORY_LIMIT=4G diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index d42c484d..36fc0451 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -11,25 +11,33 @@ jobs: build-and-push: runs-on: ubuntu-20.04 steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build and push tag if: ${{ github.ref_name != 'main' }} - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v5 with: + platforms: linux/amd64,linux/arm64 push: true tags: jembi/platform:${{ github.ref_name }} - name: Build and push latest if: ${{ github.ref_name == 'main' }} - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v5 with: + platforms: linux/amd64,linux/arm64 push: true tags: jembi/platform:latest diff --git a/Dockerfile b/Dockerfile index 87777566..8667516a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM openhie/package-base:2.3.1 +FROM openhie/package-base:2.3.2 # Install yq RUN curl -L https://github.com/mikefarah/yq/releases/download/v4.23.1/yq_linux_amd64 -o /usr/bin/yq diff --git a/README.md b/README.md index bf2dba53..1a2272ca 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,13 @@ Each service's resource allocations can be piped into their .yml file through en - Take note to not allocate less memory to ELK Stack services than their JVM heap sizes. - Exit code 137 indicates an out-of-memory failure. When running into this, it means that the service has been allocated too little memory. +## Build multi-platform docker images +It's essential to make sure that any docker image should be available for multiple platforms : AMD, ARM, ... (not only linux, but MacOS as well). To do so you can follow the steps below : +1. Create your own custom builder by running `docker buildx create --name mycustombuilder --driver docker-container --bootstrap` +2. Ask docker to use this new builder for future builds by running `docker buildx use mycustombuilder` +3. Inspect buildx to see if docker has indeed switched builders to the new one you asked it to use by running `docker buildx inspect` +4. Then you can perform the build and push, for example : `docker buildx build --platform linux/amd64,linux/arm64 --push -t jembi/hapi:v7.0.3-wget .` + ## Tests Tests are located in `/test` diff --git a/client-registry-jempi/docker-compose.api.yml b/client-registry-jempi/docker-compose.api.yml index 4cf156de..7d218748 100644 --- a/client-registry-jempi/docker-compose.api.yml +++ b/client-registry-jempi/docker-compose.api.yml @@ -33,6 +33,7 @@ services: reverse-proxy: kafka: default: + jempi: jempi-api-kc: @@ -77,6 +78,7 @@ services: reverse-proxy: kafka: default: + jempi: volumes: @@ -87,6 +89,9 @@ networks: reverse-proxy: name: reverse-proxy_public external: true + jempi: + name: jempi_public + external: true kafka: name: kafka_public external: true diff --git a/client-registry-jempi/docker-compose.combined-cluster.yml b/client-registry-jempi/docker-compose.combined-cluster.yml index ea3bd37d..2781e809 100644 --- a/client-registry-jempi/docker-compose.combined-cluster.yml +++ b/client-registry-jempi/docker-compose.combined-cluster.yml @@ -6,7 +6,13 @@ services: placement: constraints: - "node.labels.name==node-1" - + + jempi-bootstrapper: + deploy: + placement: + constraints: + - "node.labels.name==node-1" + jempi-postgresql-02: image: bitnami/postgresql-repmgr:15.2.0 environment: @@ -36,6 +42,8 @@ services: configs: - target: /docker-entrypoint-initdb.d/jempi_psql_init_db.sql source: jempi_psql_init_db.sql + networks: + jempi: jempi-postgresql-03: image: bitnami/postgresql-repmgr:15.2.0 @@ -66,7 +74,14 @@ services: configs: - target: /docker-entrypoint-initdb.d/jempi_psql_init_db.sql source: jempi_psql_init_db.sql + networks: + jempi: volumes: jempi-psql-02-data: jempi-psql-03-data: + +networks: + jempi: + name: jempi_public + external: true diff --git a/client-registry-jempi/docker-compose.combined.yml b/client-registry-jempi/docker-compose.combined.yml index 3bca3f1e..0e6a1a2b 100644 --- a/client-registry-jempi/docker-compose.combined.yml +++ b/client-registry-jempi/docker-compose.combined.yml @@ -19,6 +19,7 @@ services: networks: kafka: default: + jempi: jempi-etl: @@ -37,6 +38,7 @@ services: networks: kafka: default: + jempi: jempi-controller: @@ -70,6 +72,7 @@ services: networks: kafka: default: + jempi: jempi-linker: @@ -104,6 +107,7 @@ services: networks: kafka: default: + jempi: jempi-bootstrapper: image: jembi/jempi-bootstrapper:${JEMPI_BOOTSTRAPPER_IMAGE_TAG} @@ -124,6 +128,7 @@ services: networks: kafka: default: + jempi: jempi-postgresql-01: image: bitnami/postgresql-repmgr:15.2.0 @@ -151,6 +156,8 @@ services: configs: - target: /docker-entrypoint-initdb.d/jempi_psql_init_db.sql source: jempi_psql_init_db.sql + networks: + jempi: volumes: jempi-psql-01-data: @@ -161,7 +168,10 @@ networks: kafka: name: kafka_public external: true - defualt: + default: + jempi: + name: jempi_public + external: true configs: diff --git a/client-registry-jempi/docker-compose.dgraph-zero-cluster.yml b/client-registry-jempi/docker-compose.dgraph-zero-cluster.yml index 853f7c7f..fe7e61f8 100644 --- a/client-registry-jempi/docker-compose.dgraph-zero-cluster.yml +++ b/client-registry-jempi/docker-compose.dgraph-zero-cluster.yml @@ -7,7 +7,9 @@ services: constraints: - node.labels.name == node-1 command: dgraph zero --my=jempi-zero-01:5080 --replicas 3 --bindall --raft "idx=1" - + networks: + jempi: + jempi-zero-02: image: dgraph/dgraph:v22.0.0 volumes: @@ -25,6 +27,8 @@ services: restart_policy: condition: on-failure command: dgraph zero --my=jempi-zero-02:5080 --replicas 3 --peer=jempi-zero-01:5080 --raft "idx=2" + networks: + jempi: jempi-zero-03: image: dgraph/dgraph:v22.0.0 @@ -43,7 +47,14 @@ services: restart_policy: condition: on-failure command: dgraph zero --my=jempi-zero-03:5080 --replicas 3 --peer=jempi-zero-01:5080 --raft "idx=3" + networks: + jempi: volumes: jempi-zero-02-data: jempi-zero-03-data: + +networks: + jempi: + name: jempi_public + external: true diff --git a/client-registry-jempi/docker-compose.dgraph-zero.yml b/client-registry-jempi/docker-compose.dgraph-zero.yml index ac1974e3..8b8da6d0 100644 --- a/client-registry-jempi/docker-compose.dgraph-zero.yml +++ b/client-registry-jempi/docker-compose.dgraph-zero.yml @@ -15,6 +15,13 @@ services: restart_policy: condition: on-failure command: dgraph zero --my=jempi-zero-01:5080 --replicas 1 + networks: + jempi: volumes: jempi-zero-01-data: + +networks: + jempi: + name: jempi_public + external: true diff --git a/client-registry-jempi/docker-compose.dgraph.yml b/client-registry-jempi/docker-compose.dgraph.yml index 296aff7d..93ada3f7 100644 --- a/client-registry-jempi/docker-compose.dgraph.yml +++ b/client-registry-jempi/docker-compose.dgraph.yml @@ -16,6 +16,8 @@ services: restart_policy: condition: on-failure command: dgraph alpha --my=jempi-alpha-01:7080 --zero=jempi-zero-01:5080 --security whitelist=0.0.0.0/0 --telemetry "sentry=false;" + networks: + jempi: jempi-alpha-02: image: dgraph/dgraph:v22.0.0 @@ -32,6 +34,8 @@ services: restart_policy: condition: on-failure command: dgraph alpha --my=jempi-alpha-02:7081 --zero=jempi-zero-01:5080 --security whitelist=0.0.0.0/0 -o 1 --telemetry "sentry=false;" + networks: + jempi: jempi-alpha-03: image: dgraph/dgraph:v22.0.0 @@ -48,6 +52,8 @@ services: restart_policy: condition: on-failure command: dgraph alpha --my=jempi-alpha-03:7082 --zero=jempi-zero-01:5080 --security whitelist=0.0.0.0/0 -o 2 --telemetry "sentry=false;" + networks: + jempi: jempi-ratel: image: dgraph/ratel:v21.03.2 @@ -61,8 +67,15 @@ services: restart_policy: condition: on-failure command: dgraph-ratel + networks: + jempi: volumes: jempi-alpha-01-data: jempi-alpha-02-data: jempi-alpha-03-data: + +networks: + jempi: + name: jempi_public + external: true diff --git a/client-registry-jempi/importer/mapping-mediator/searchAll.json b/client-registry-jempi/importer/mapping-mediator/searchAll.json index 49134fca..c555e89f 100644 --- a/client-registry-jempi/importer/mapping-mediator/searchAll.json +++ b/client-registry-jempi/importer/mapping-mediator/searchAll.json @@ -9,7 +9,7 @@ }, "inputTransforms": { "total": "$count(lookupRequests.jempiSearchAll.data.goldenRecords)", - "entry": "$map(lookupRequests.jempiSearchAll.data.goldenRecords, function($v) {{'fullUrl': 'Patient/' & $v.goldenId, 'resource': {'resourceType': 'Patient','id': $v.goldenId,'name': {'given': [$v.demographicData.givenName],'family': $v.demographicData.familyName},'address': [{'city': $v.demographicData.city}],'birthDate': $v.demographicData.dob,'telecom': [{'value': $v.demographicData.phoneNumber,'system': 'phone'}],'identifier': [{'system': $v.sourceId.facility,'value': $v.sourceId.patient},{'system': 'NationalID','value': $v.demographicData.nationalId}],'gender': $v.demographicData.gender}}})" + "entry": "$append([], $map(lookupRequests.jempiSearchAll.data.goldenRecords, function($v) {{'fullUrl': 'Patient/' & $v.goldenId, 'resource': {'resourceType': 'Patient','id': $v.goldenId,'name': {'given': [$v.demographicData.givenName],'family': $v.demographicData.familyName},'address': [{'city': $v.demographicData.city}],'birthDate': $v.demographicData.dob,'telecom': [{'value': $v.demographicData.phoneNumber,'system': 'phone'}],'identifier': [{'system': $v.sourceId.facility,'value': $v.sourceId.patient},{'system': 'NationalID','value': $v.demographicData.nationalId}],'gender': $v.demographicData.gender}}}))" }, "inputMapping": { "constants.resourceType": "resourceType", diff --git a/config.yaml b/config.yaml index e561e5a9..638db7b1 100644 --- a/config.yaml +++ b/config.yaml @@ -24,6 +24,7 @@ packages: - identity-access-manager-keycloak - openhim-mapping-mediator - database-postgres + - reprocess-mediator - fhir-ig-importer profiles: @@ -43,6 +44,8 @@ profiles: - openhim-mapping-mediator - kafka-mapper-consumer - kafka-unbundler-consumer + - fhir-ig-importer + - reprocess-mediator envFiles: - cdr-dw.env diff --git a/documentation/recipes/README.md b/documentation/recipes/README.md index 5382f24f..b10d3a91 100644 --- a/documentation/recipes/README.md +++ b/documentation/recipes/README.md @@ -1,13 +1,12 @@ + --- description: Pre-defined recipes for common use cases --- # 📜 Recipes -OpenHIM platform comes bundles with a set of generic packages that can be deployed and configured to support a number of different use cases. To help users of OpenHIM Platform get started with something they can make use of immediately, a number of default OpenHIM Platform reciepes are provided. These help you get started with everything you need setup and configured for a particular use case. - -We current support the following default recipes: - +OpenHIM platform comes bundled with a set of generic packages that can be deployed and configured to support a number of different use cases. To help users of OpenHIM Platform get started with something they can make use of immediately, a number of default OpenHIM Platform recipes are provided. These help you get started with everything you need setup and configured for a particular use case. +We currently support the following default recipes:
Central Data Repository with Data WarehouseA FHIR-based Shared Health record linked to an MPI for linking and matching patient demographics and a default reporting pipeline to transform and visualise FHIR data.central-data-repository-with-data-warehousing.md
Central Data RepositoryA FHIR-based Shared Health record linked to an MPI for linking and matching patient demographics. No reporting is include but all FHIR data is pushed to Kafka for external system to use.central-data-repository-no-reporting.md
Master Patient IndexA master patient index setup using JeMPI. it also includes OpenHIM as the API gateway providing security, a mapping mediator to allow FHIR-based communication with JeMPI and Keycloak to support user management.master-patient-index.md
diff --git a/fhir-ig-importer/docker-compose.dev.yml b/fhir-ig-importer/docker-compose.dev.yml index 25714fbd..c33a9169 100644 --- a/fhir-ig-importer/docker-compose.dev.yml +++ b/fhir-ig-importer/docker-compose.dev.yml @@ -10,5 +10,5 @@ services: fhir-ig-importer-ui: ports: - target: 8080 - published: 3000 + published: 3334 mode: host diff --git a/fhir-ig-importer/docker-compose.yml b/fhir-ig-importer/docker-compose.yml index 43c0be1a..1faf0253 100644 --- a/fhir-ig-importer/docker-compose.yml +++ b/fhir-ig-importer/docker-compose.yml @@ -6,6 +6,7 @@ services: networks: hapi-fhir: openhim: + default: environment: HAPI_FHIR_BASE_URL: ${HAPI_FHIR_BASE_URL} HAPI_FHIR_INSTANCES: ${HAPI_FHIR_INSTANCES} @@ -27,4 +28,5 @@ networks: openhim: name: openhim_public external: true + default: diff --git a/fhir-ig-importer/importer/docker-compose.config.yml b/fhir-ig-importer/importer/docker-compose.config.yml index 2f47ef17..0d11921a 100644 --- a/fhir-ig-importer/importer/docker-compose.config.yml +++ b/fhir-ig-importer/importer/docker-compose.config.yml @@ -18,6 +18,8 @@ services: target: /openhimConfig.js - source: fhir-ig-importer-config-importer-openhim-import.json target: /openhim-import.json + - source: fhir-ig-importer-config-importer-ig-importer-app.json + target: /ig-importer-app.json deploy: replicas: 1 restart_policy: @@ -34,8 +36,14 @@ configs: name: fhir-ig-importer-config-importer-openhim-import.json-${fhir_ig_importer_config_importer_openhim_import_js_DIGEST:?err} labels: name: fhir-ig-importer + fhir-ig-importer-config-importer-ig-importer-app.json: + file: ./volume/ig-importer-app.json + name: fhir-ig-importer-config-importer-ig-importer-app.json-${fhir_ig_importer_config_importer_ig_importer_app_DIGEST:?err} + labels: + name: fhir-ig-importer networks: openhim: name: openhim_public external: true + default: diff --git a/fhir-ig-importer/importer/volume/ig-importer-app.json b/fhir-ig-importer/importer/volume/ig-importer-app.json new file mode 100644 index 00000000..102b6eac --- /dev/null +++ b/fhir-ig-importer/importer/volume/ig-importer-app.json @@ -0,0 +1,11 @@ +{ + "name": "FHIR IG Importer", + "description": "FHIR IG microfrontend app", + "category": "HIE Configuration", + "type": "esmodule", + "url": "http://localhost:3334/jembi-fhir-ig-importer.js", + "showInPortal": true, + "showInSideBar": true, + "access_roles": ["admin"], + "icon": "https://fonts.gstatic.com/s/i/materialicons/medical_information/v1/24px.svg" +} diff --git a/fhir-ig-importer/importer/volume/openhimConfig.js b/fhir-ig-importer/importer/volume/openhimConfig.js index 182ee887..cae7a270 100644 --- a/fhir-ig-importer/importer/volume/openhimConfig.js +++ b/fhir-ig-importer/importer/volume/openhimConfig.js @@ -1,53 +1,90 @@ -"use strict"; - const fs = require("fs"); const https = require("https"); const path = require("path"); +("use strict"); + const OPENHIM_CORE_SERVICE_NAME = "openhim-core"; const OPENHIM_MEDIATOR_API_PORT = 8080; const OPENHIM_API_PASSWORD = process.env.OPENHIM_API_PASSWORD || "instant101"; const OPENHIM_API_USERNAME = process.env.OPENHIM_API_USERNAME || "root@openhim.org"; -const authHeader = new Buffer.from( +const authHeader = Buffer.from( `${OPENHIM_API_USERNAME}:${OPENHIM_API_PASSWORD}` ).toString("base64"); +function makeRequest(options, data) { + const req = https.request(options, (res) => { + if (res.statusCode == 401) { + throw new Error(`Incorrect OpenHIM API credentials`); + } + + if (![201, 200].includes(res.statusCode)) { + throw new Error(`Failed to import OpenHIM config: ${res.statusCode}`); + } + + console.log("Successfully Imported OpenHIM Config"); + }); + + req.on("error", (error) => { + throw new Error(`Failed to import OpenHIM config: ${error}`); + }); + + req.write(data); + req.end(); +} + const jsonData = JSON.parse( fs.readFileSync(path.resolve(__dirname, "openhim-import.json")) ); +const appJsonData = JSON.parse( + fs.readFileSync(path.resolve(__dirname, "ig-importer-app.json")) +); + const data = JSON.stringify(jsonData); +const appData = JSON.stringify(appJsonData); const options = { protocol: "https:", hostname: OPENHIM_CORE_SERVICE_NAME, port: OPENHIM_MEDIATOR_API_PORT, - path: "/metadata", - method: "POST", headers: { "Content-Type": "application/json", - "Content-Length": data.length, Authorization: `Basic ${authHeader}`, }, }; -const req = https.request(options, (res) => { - if (res.statusCode == 401) { - throw new Error(`Incorrect OpenHIM API credentials`); - } - - if (res.statusCode != 201) { - throw new Error(`Failed to import OpenHIM config: ${res.statusCode}`); - } +const reqOptions = { + ...options, + path: "/metadata", + method: "POST", + headers: { + ...options.headers, + "Content-Length": data.length, + }, +}; - console.log("Successfully Imported OpenHIM Config"); -}); +const appReqOptions = { + ...options, + path: "/apps", + method: "POST", + headers: { + ...options.headers, + "Content-Length": appData.length, + }, +}; -req.on("error", (error) => { - throw new Error(`Failed to import OpenHIM config: ${error}`); -}); +const importMapRebuildOptions = { + ...options, + path: "/apps", + method: "GET", + headers: { + ...options.headers, + }, +}; -req.write(data); -req.end(); +makeRequest(reqOptions, data); +makeRequest(appReqOptions, appData); +makeRequest(importMapRebuildOptions, ""); diff --git a/fhir-ig-importer/package-metadata.json b/fhir-ig-importer/package-metadata.json index 1db85f0a..27f4de82 100644 --- a/fhir-ig-importer/package-metadata.json +++ b/fhir-ig-importer/package-metadata.json @@ -14,7 +14,7 @@ "FHIR_IG_IMPORTER_CORE_PORT": 3001, "FHIR_IG_IMPORTER_CORE_HOST": "0.0.0.0", "FHIR_IG_IMPORTER_CORE_URL": "http://0.0.0.0:3001/fhir/ig/v1.0", - "FHIR_IG_IMPORTER_UI_VERSION": "0.1.0", - "FHIR_IG_IMPORTER_CORE_VERSION": "1.0.0" + "FHIR_IG_IMPORTER_UI_VERSION": "v1.0.0", + "FHIR_IG_IMPORTER_CORE_VERSION": "v1.0.0" } } diff --git a/identity-access-manager-keycloak/docker-compose-postgres.cluster.yml b/identity-access-manager-keycloak/docker-compose-postgres.cluster.yml deleted file mode 100644 index a6ec7061..00000000 --- a/identity-access-manager-keycloak/docker-compose-postgres.cluster.yml +++ /dev/null @@ -1,75 +0,0 @@ -version: '3.9' - -services: - keycloak-postgres-1: - environment: - REPMGR_PARTNER_NODES: ${KC_REPMGR_PARTNER_NODES} - deploy: - placement: - constraints: - - "node.labels.name==node-1" - - keycloak-postgres-2: - image: bitnami/postgresql-repmgr:14 - environment: - POSTGRESQL_PASSWORD: ${KC_POSTGRESQL_PASSWORD} - POSTGRESQL_USERNAME: ${KC_POSTGRESQL_USERNAME} - POSTGRESQL_DATABASE: ${KC_POSTGRESQL_DATABASE} - REPMGR_NODE_NETWORK_NAME: keycloak-postgres-2 - REPMGR_PASSWORD: ${KC_REPMGR_PASSWORD} - REPMGR_RECONNECT_INTERVAL: 3 - REPMGR_NODE_NAME: keycloak-postgres-2 - REPMGR_PRIMARY_HOST: ${KC_REPMGR_PRIMARY_HOST} - REPMGR_PARTNER_NODES: ${KC_REPMGR_PARTNER_NODES} - volumes: - - 'keycloak-postgres-2-data:/bitnami/postgresql' - deploy: - placement: - constraints: - - "node.labels.name==node-2" - replicas: 1 - resources: - limits: - cpus: ${KC_POSTGRES_CPU_LIMIT} - memory: ${KC_POSTGRES_MEMORY_LIMIT} - reservations: - cpus: ${KC_POSTGRES_CPU_RESERVE} - memory: ${KC_POSTGRES_MEMORY_RESERVE} - networks: - default: - keycloak_backup_net: {} - - - keycloak-postgres-3: - image: bitnami/postgresql-repmgr:14 - environment: - POSTGRESQL_PASSWORD: ${KC_POSTGRESQL_PASSWORD} - POSTGRESQL_USERNAME: ${KC_POSTGRESQL_USERNAME} - POSTGRESQL_DATABASE: ${KC_POSTGRESQL_DATABASE} - REPMGR_NODE_NETWORK_NAME: keycloak-postgres-3 - REPMGR_PASSWORD: ${KC_REPMGR_PASSWORD} - REPMGR_RECONNECT_INTERVAL: 3 - REPMGR_NODE_NAME: keycloak-postgres-3 - REPMGR_PRIMARY_HOST: ${KC_REPMGR_PRIMARY_HOST} - REPMGR_PARTNER_NODES: ${KC_REPMGR_PARTNER_NODES} - volumes: - - 'keycloak-postgres-3-data:/bitnami/postgresql' - deploy: - placement: - constraints: - - "node.labels.name==node-3" - replicas: 1 - resources: - limits: - cpus: ${KC_POSTGRES_CPU_LIMIT} - memory: ${KC_POSTGRES_MEMORY_LIMIT} - reservations: - cpus: ${KC_POSTGRES_CPU_RESERVE} - memory: ${KC_POSTGRES_MEMORY_RESERVE} - networks: - default: - keycloak_backup_net: {} - -volumes: - keycloak-postgres-2-data: - keycloak-postgres-3-data: diff --git a/identity-access-manager-keycloak/docker-compose-postgres.dev.yml b/identity-access-manager-keycloak/docker-compose-postgres.dev.yml deleted file mode 100644 index 79e221ae..00000000 --- a/identity-access-manager-keycloak/docker-compose-postgres.dev.yml +++ /dev/null @@ -1,8 +0,0 @@ -version: '3.9' - -services: - keycloak-postgres-1: - ports: - - target: 5432 - published: 5434 - mode: host diff --git a/identity-access-manager-keycloak/docker-compose-postgres.yml b/identity-access-manager-keycloak/docker-compose-postgres.yml deleted file mode 100644 index 4a8f2279..00000000 --- a/identity-access-manager-keycloak/docker-compose-postgres.yml +++ /dev/null @@ -1,40 +0,0 @@ -version: "3.9" - -services: - keycloak-postgres-1: - image: bitnami/postgresql-repmgr:14 - environment: - POSTGRESQL_PASSWORD: ${KC_POSTGRESQL_PASSWORD} - POSTGRESQL_USERNAME: ${KC_POSTGRESQL_USERNAME} - POSTGRESQL_DATABASE: ${KC_POSTGRESQL_DATABASE} - POSTGRESQL_POSTGRES_PASSWORD: ${KC_POSTGRESQL_PASSWORD} - REPMGR_NODE_NETWORK_NAME: keycloak-postgres-1 - REPMGR_PASSWORD: ${KC_REPMGR_PASSWORD} - REPMGR_RECONNECT_INTERVAL: 3 - REPMGR_NODE_NAME: keycloak-postgres-1 - REPMGR_PRIMARY_HOST: ${KC_REPMGR_PRIMARY_HOST} - REPMGR_PARTNER_NODES: ${KC_REPMGR_PARTNER_NODES} - volumes: - - "keycloak-postgres-1-data:/bitnami/postgresql" - deploy: - replicas: 1 - resources: - limits: - cpus: ${KC_POSTGRES_CPU_LIMIT} - memory: ${KC_POSTGRES_MEMORY_LIMIT} - reservations: - cpus: ${KC_POSTGRES_CPU_RESERVE} - memory: ${KC_POSTGRES_MEMORY_RESERVE} - networks: - default: - keycloak_backup_net: {} - -volumes: - keycloak-postgres-1-data: - -networks: - default: - keycloak_backup_net: - name: keycloak_backup - driver: overlay - attachable: true diff --git a/identity-access-manager-keycloak/docker-compose.yml b/identity-access-manager-keycloak/docker-compose.yml index bb4d38dc..d2025ac4 100644 --- a/identity-access-manager-keycloak/docker-compose.yml +++ b/identity-access-manager-keycloak/docker-compose.yml @@ -1,14 +1,14 @@ -version: '3.9' +version: "3.9" services: identity-access-manager-keycloak: - image: keycloak/keycloak:20.0 + image: ${KEYCLOAK_IMAGE} command: [ "start", "--proxy=edge", "--hostname-url=${KC_FRONTEND_URL}", - "--import-realm" + "--import-realm", ] hostname: identity-access-manager-keycloak healthcheck: @@ -44,7 +44,7 @@ services: KC_OPENHIM_ROOT_URL: ${KC_OPENHIM_ROOT_URL} deploy: placement: - max_replicas_per_node: 1 + max_replicas_per_node: ${KEYCLOAK_MAX_REPLICAS_PER_NODE} networks: reverse-proxy: public: diff --git a/identity-access-manager-keycloak/package-metadata.json b/identity-access-manager-keycloak/package-metadata.json index 6b6436b1..88d546cb 100644 --- a/identity-access-manager-keycloak/package-metadata.json +++ b/identity-access-manager-keycloak/package-metadata.json @@ -6,6 +6,14 @@ "version": "0.0.1", "dependencies": ["database-postgres"], "environmentVariables": { + "POSTGRES_IMAGE": "bitnami/postgresql-repmgr:14", + "KEYCLOAK_POSTGRES_1": "node-1", + "KEYCLOAK_POSTGRES_2": "node-2", + "KEYCLOAK_POSTGRES_3": "node-3", + "POSTGRES_REPLICAS": "1", + "KEYCLOAK_IMAGE": "keycloak/keycloak:20.0", + "KEYCLOAK_REPLICAS": "1", + "KEYCLOAK_MAX_REPLICAS_PER_NODE": "1", "KEYCLOAK_ADMIN": "admin", "KEYCLOAK_ADMIN_PASSWORD": "dev_password_only", "KC_FRONTEND_URL": "http://localhost:9088", diff --git a/interoperability-layer-openhim/docker-compose-mongo.cluster.yml b/interoperability-layer-openhim/docker-compose-mongo.cluster.yml index 566ec6b5..79868fdf 100644 --- a/interoperability-layer-openhim/docker-compose-mongo.cluster.yml +++ b/interoperability-layer-openhim/docker-compose-mongo.cluster.yml @@ -1,23 +1,23 @@ -version: '3.9' +version: "3.9" services: mongo-1: - command: ['--replSet', 'mongo-set', '--wiredTigerCacheSizeGB', '0.5'] + command: ["--replSet", "mongo-set", "--wiredTigerCacheSizeGB", "0.5"] deploy: placement: constraints: - - "node.labels.name==node-1" + - "node.labels.name==${MONGO_1_PLACEMENT}" mongo-2: - image: mongo:4.2 + image: ${MONGO_IMAGE} volumes: - - 'openhim-mongo-02:/data/db' - - 'openhim-mongo-02-config:/data/configdb' - command: ['--replSet', 'mongo-set', '--wiredTigerCacheSizeGB', '0.5'] + - "openhim-mongo-02:/data/db" + - "openhim-mongo-02-config:/data/configdb" + command: ["--replSet", "mongo-set", "--wiredTigerCacheSizeGB", "0.5"] deploy: placement: constraints: - - "node.labels.name==node-2" + - "node.labels.name==${MONGO_2_PLACEMENT}" replicas: 1 resources: limits: @@ -32,15 +32,15 @@ services: mongo_backup_net: mongo-3: - image: mongo:4.2 + image: ${MONGO_IMAGE} volumes: - - 'openhim-mongo-03:/data/db' - - 'openhim-mongo-03-config:/data/configdb' - command: ['--replSet', 'mongo-set', '--wiredTigerCacheSizeGB', '0.5'] + - "openhim-mongo-03:/data/db" + - "openhim-mongo-03-config:/data/configdb" + command: ["--replSet", "mongo-set", "--wiredTigerCacheSizeGB", "0.5"] deploy: placement: constraints: - - "node.labels.name==node-3" + - "node.labels.name==${MONGO_3_PLACEMENT}" replicas: 1 resources: limits: diff --git a/interoperability-layer-openhim/docker-compose-mongo.yml b/interoperability-layer-openhim/docker-compose-mongo.yml index c67589f6..69117ee7 100644 --- a/interoperability-layer-openhim/docker-compose-mongo.yml +++ b/interoperability-layer-openhim/docker-compose-mongo.yml @@ -2,8 +2,8 @@ version: "3.9" services: mongo-1: - image: mongo:4.2 - command: ['--replSet', 'mongo-set', '--wiredTigerCacheSizeGB', '0.5'] + image: ${MONGO_IMAGE} + command: ["--replSet", "mongo-set", "--wiredTigerCacheSizeGB", "0.5"] volumes: - "openhim-mongo-01:/data/db" - "openhim-mongo-01-config:/data/configdb" diff --git a/interoperability-layer-openhim/docker-compose.await-helper-mongo.yml b/interoperability-layer-openhim/docker-compose.await-helper-mongo.yml index c02d4ffe..82729d17 100644 --- a/interoperability-layer-openhim/docker-compose.await-helper-mongo.yml +++ b/interoperability-layer-openhim/docker-compose.await-helper-mongo.yml @@ -1,10 +1,10 @@ -version: '3.9' +version: "3.9" services: await-helper: - image: jembi/await-helper:1.0.1 + image: ${AWAIT_HELPER_IMAGE} deploy: replicas: 1 restart_policy: condition: none - command: '-k http://mongo-1:27017' + command: "-k http://mongo-1:27017" diff --git a/interoperability-layer-openhim/docker-compose.await-helper.yml b/interoperability-layer-openhim/docker-compose.await-helper.yml index 8fe4c43e..9625820b 100644 --- a/interoperability-layer-openhim/docker-compose.await-helper.yml +++ b/interoperability-layer-openhim/docker-compose.await-helper.yml @@ -1,10 +1,10 @@ -version: '3.9' +version: "3.9" services: await-helper: - image: jembi/await-helper:1.0.1 + image: ${AWAIT_HELPER_IMAGE} deploy: replicas: 1 restart_policy: condition: none - command: '-k https://openhim-core:8080/heartbeat' + command: "-k https://openhim-core:8080/heartbeat" diff --git a/interoperability-layer-openhim/docker-compose.yml b/interoperability-layer-openhim/docker-compose.yml index 0728edc2..1413642a 100644 --- a/interoperability-layer-openhim/docker-compose.yml +++ b/interoperability-layer-openhim/docker-compose.yml @@ -11,6 +11,7 @@ services: public: default: prometheus: + reprocess: environment: - mongo_url=${OPENHIM_MONGO_URL} - mongo_atnaUrl=${OPENHIM_MONGO_ATNAURL} @@ -22,10 +23,11 @@ services: - api_openid_callbackUrl=${KC_OPENHIM_ROOT_URL} - api_openid_clientId=${KC_OPENHIM_CLIENT_ID} - api_openid_clientSecret=${KC_OPENHIM_CLIENT_SECRET} + - openhimConsoleBaseUrl=${OPENHIM_CONSOLE_BASE_URL} deploy: replicas: ${OPENHIM_CORE_INSTANCES} placement: - max_replicas_per_node: 1 + max_replicas_per_node: ${OPENHIM_CORE_MAX_REPLICAS_PER_NODE} resources: limits: cpus: ${OPENHIM_CPU_LIMIT} @@ -39,7 +41,7 @@ services: - prometheus-port=8080 openhim-console: - image: ${OPENHIM_CONSOLE_VERSION} + image: ${OPENHIM_CONSOLE_IMAGE} environment: OPENHIM_CORE_MEDIATOR_HOSTNAME: ${OPENHIM_CORE_MEDIATOR_HOSTNAME} OPENHIM_MEDIATOR_API_PORT: ${OPENHIM_MEDIATOR_API_PORT} @@ -48,6 +50,8 @@ services: KC_REALM_NAME: ${KC_REALM_NAME} KC_FRONTEND_URL: ${KC_FRONTEND_URL} OPENHIM_CONSOLE_SHOW_LOGIN: ${OPENHIM_CONSOLE_SHOW_LOGIN} + REACT_APP_OPENHIM_API_BASE_URL: ${OPENHIM_API_BASE_URL} + NODE_TLS_REJECT_UNAUTHORIZED: 0 networks: reverse-proxy: keycloak: @@ -56,7 +60,7 @@ services: deploy: replicas: ${OPENHIM_CONSOLE_INSTANCES} placement: - max_replicas_per_node: 1 + max_replicas_per_node: ${OPENHIM_CONSOLE_MAX_REPLICAS_PER_NODE} resources: limits: cpus: ${OPENHIM_CONSOLE_CPU_LIMIT} @@ -84,4 +88,7 @@ networks: prometheus: name: prometheus_public external: true + reprocess: + name: reprocess_public + external: true default: diff --git a/interoperability-layer-openhim/package-metadata.json b/interoperability-layer-openhim/package-metadata.json index 9bfef8e2..82272031 100644 --- a/interoperability-layer-openhim/package-metadata.json +++ b/interoperability-layer-openhim/package-metadata.json @@ -6,10 +6,18 @@ "type": "infrastructure", "dependencies": [], "environmentVariables": { - "OPENHIM_CORE_IMAGE": "jembi/openhim-core:microfrontends-pre-release", + "OPENHIM_CORE_IMAGE": "jembi/openhim-core:v8.4.3", + "OPENHIM_CONSOLE_IMAGE": "jembi/openhim-console:poc-microfrontend", + "MONGO_IMAGE": "mongo:4.2", + "AWAIT_HELPER_IMAGE": "jembi/await-helper:1.0.1", + "MONGO_1_PLACEMENT": "node-1", + "MONGO_2_PLACEMENT": "node-2", + "MONGO_3_PLACEMENT": "node-3", "MONGO_SET_COUNT": "1", "OPENHIM_CORE_INSTANCES": "1", + "OPENHIM_CORE_MAX_REPLICAS_PER_NODE": "1", "OPENHIM_CONSOLE_INSTANCES": "1", + "OPENHIM_CONSOLE_MAX_REPLICAS_PER_NODE": "1", "OPENHIM_CORE_MEDIATOR_HOSTNAME": "localhost", "OPENHIM_MEDIATOR_API_PORT": "8080", "OPENHIM_CPU_LIMIT": "0", @@ -27,7 +35,6 @@ "OPENHIM_MONGO_MEMORY_RESERVE": "500M", "OPENHIM_MONGO_URL": "mongodb://mongo-1:27017/openhim", "OPENHIM_MONGO_ATNAURL": "mongodb://mongo-1:27017/openhim", - "OPENHIM_CONSOLE_VERSION": "jembi/openhim-console:microfrontend-poc", "KAFKA_HOSTS": "kafka-01:9092", "KC_REALM_NAME": "platform-realm", "KC_FRONTEND_URL": "http://localhost:9088", diff --git a/job-scheduler-ofelia/docker-compose.yml b/job-scheduler-ofelia/docker-compose.yml index 5cd0e3c8..00740f61 100644 --- a/job-scheduler-ofelia/docker-compose.yml +++ b/job-scheduler-ofelia/docker-compose.yml @@ -1,8 +1,8 @@ -version: '3.9' +version: "3.9" services: job-scheduler-ofelia: - image: mcuadros/ofelia:v0.3.6 + image: ${JOB_SCHEDULER_OFELIA_IMAGE} command: daemon --config=/tmp/config.ini configs: - target: /tmp/config.ini @@ -14,5 +14,5 @@ configs: ofelia-config.ini: file: ./config.ini name: ofelia-config.ini-${ofelia_config_ini_DIGEST:?err} - labels: + labels: name: ofelia diff --git a/job-scheduler-ofelia/package-metadata.json b/job-scheduler-ofelia/package-metadata.json index a4592a6d..719dffae 100644 --- a/job-scheduler-ofelia/package-metadata.json +++ b/job-scheduler-ofelia/package-metadata.json @@ -6,6 +6,7 @@ "version": "0.0.1", "dependencies": [], "environmentVariables": { + "JOB_SCHEDULER_OFELIA_IMAGE": "mcuadros/ofelia:v0.3.6", "RENEWAL_EMAIL": "dummy@jembi.org", "STAGING": "true", "DOMAIN_NAME": "localhost", diff --git a/kafka-mapper-consumer/consumer-ui-app.json b/kafka-mapper-consumer/consumer-ui-app.json new file mode 100644 index 00000000..b6ac02d7 --- /dev/null +++ b/kafka-mapper-consumer/consumer-ui-app.json @@ -0,0 +1,11 @@ +{ + "name": "Kafka Mapper Consumer", + "description": "Kafka mapper consumer microfrontends app", + "category": "HIE Configuration", + "type": "esmodule", + "url": "http://localhost:8091/jembi-kafka-mapper-consumer-ui.js", + "showInPortal": true, + "showInSideBar": false, + "access_roles": ["admin"], + "icon": "https://fonts.gstatic.com/s/i/materialicons/apps/v12/24px.svg" +} diff --git a/kafka-mapper-consumer/docker-compose.config.yml b/kafka-mapper-consumer/docker-compose.config.yml new file mode 100644 index 00000000..2e1b4db5 --- /dev/null +++ b/kafka-mapper-consumer/docker-compose.config.yml @@ -0,0 +1,42 @@ +version: '3.9' + +services: + # container for executing config import scripts for creating the OpenHIM channels used by the Mediator + kafka-mapper-consumer-config-importer: + image: node:erbium-alpine + networks: + openhim: + default: + environment: + OPENHIM_API_USERNAME: ${OPENHIM_USERNAME} + OPENHIM_API_PASSWORD: ${OPENHIM_PASSWORD} + # Reject unauthorised is only needed if the OpenHIM's SSL is not setup + NODE_TLS_REJECT_UNAUTHORIZED: 0 + command: sh -c "node openhimConfig.js" + configs: + - source: kafka-mapper-consumer-openhimConfig.js + target: /openhimConfig.js + - source: kafka-mapper-consumer-consumer-ui-app.json + target: /consumer-ui-app.json + deploy: + replicas: 1 + restart_policy: + condition: none + +configs: + kafka-mapper-consumer-openhimConfig.js: + file: ./openhimConfig.js + name: kafka-mapper-consumer-openhimConfig.js-${fhir_ig_importer_config_importer_openhimConfig_js_DIGEST:?err} + labels: + name: kafka-mapper-consumer + kafka-mapper-consumer-consumer-ui-app.json: + file: ./consumer-ui-app.json + name: kafka-mapper-consumer-consumer-ui-app.json-${kafka_mapper_consumer_ui_json_DIGEST:?err} + labels: + name: kafka-mapper-consumer + +networks: + openhim: + name: openhim_public + external: true + default: diff --git a/kafka-mapper-consumer/docker-compose.yml b/kafka-mapper-consumer/docker-compose.yml index c146a75c..cc031e00 100644 --- a/kafka-mapper-consumer/docker-compose.yml +++ b/kafka-mapper-consumer/docker-compose.yml @@ -21,7 +21,8 @@ services: networks: clickhouse: kafka: - + openhim: + default: kafka-mapper-consumer-ui: image: ${KAFKA_CONSUMER_MAPPER_UI_VERSION} networks: @@ -41,3 +42,7 @@ networks: kafka: name: kafka_public external: true + openhim: + name: openhim_public + external: true + default: diff --git a/kafka-mapper-consumer/openhimConfig.js b/kafka-mapper-consumer/openhimConfig.js new file mode 100644 index 00000000..a7868cde --- /dev/null +++ b/kafka-mapper-consumer/openhimConfig.js @@ -0,0 +1,72 @@ +const fs = require("fs"); +const https = require("https"); +const path = require("path"); + +("use strict"); + +const OPENHIM_CORE_SERVICE_NAME = "openhim-core"; +const OPENHIM_MEDIATOR_API_PORT = 8080; +const OPENHIM_API_PASSWORD = process.env.OPENHIM_API_PASSWORD || "instant101"; +const OPENHIM_API_USERNAME = + process.env.OPENHIM_API_USERNAME || "root@openhim.org"; + +const authHeader = Buffer.from( + `${OPENHIM_API_USERNAME}:${OPENHIM_API_PASSWORD}` +).toString("base64"); +function makeRequest(options, data) { + const req = https.request(options, (res) => { + if (res.statusCode == 401) { + throw new Error(`Incorrect OpenHIM API credentials`); + } + + if (![201, 200].includes(res.statusCode)) { + throw new Error(`Failed to import OpenHIM config: ${res.statusCode}`); + } + + console.log("Successfully Imported OpenHIM Config"); + }); + + req.on("error", (error) => { + throw new Error(`Failed to import OpenHIM config: ${error}`); + }); + + req.write(data); + req.end(); +} + +const appJsonData = JSON.parse( + fs.readFileSync(path.resolve(__dirname, "consumer-ui-app.json")) +); +const appData = JSON.stringify(appJsonData); + +const options = { + protocol: "https:", + hostname: OPENHIM_CORE_SERVICE_NAME, + port: OPENHIM_MEDIATOR_API_PORT, + headers: { + "Content-Type": "application/json", + Authorization: `Basic ${authHeader}`, + }, +}; + +const appReqOptions = { + ...options, + path: "/apps", + method: "POST", + headers: { + ...options.headers, + "Content-Length": appData.length, + }, +}; + +const importMapRebuildOptions = { + ...options, + path: "/apps", + method: "GET", + headers: { + ...options.headers, + }, +}; + +makeRequest(appReqOptions, appData); +makeRequest(importMapRebuildOptions, ""); diff --git a/kafka-mapper-consumer/package-metadata.json b/kafka-mapper-consumer/package-metadata.json index 57fc4199..0c0b17a9 100644 --- a/kafka-mapper-consumer/package-metadata.json +++ b/kafka-mapper-consumer/package-metadata.json @@ -17,7 +17,7 @@ "REGISTER_MEDIATOR": "true", "CLICKHOUSE_HOST": "analytics-datastore-clickhouse", "CLICKHOUSE_PORT": "8123", - "KAFKA_CONSUMER_MAPPER_MEDIATOR_VERSION": "jembi/kafka-mapper-consumer:0.1.0", - "KAFKA_CONSUMER_MAPPER_UI_VERSION": "jembi/kafka-mapper-consumer-ui:0.1.0-alpha" + "KAFKA_CONSUMER_MAPPER_MEDIATOR_VERSION": "jembi/kafka-mapper-consumer:v0.0.1", + "KAFKA_CONSUMER_MAPPER_UI_VERSION": "jembi/kafka-mapper-consumer-ui:v0.0.1" } } diff --git a/kafka-mapper-consumer/swarm.sh b/kafka-mapper-consumer/swarm.sh index 8d1f9ed6..dbf5e83d 100755 --- a/kafka-mapper-consumer/swarm.sh +++ b/kafka-mapper-consumer/swarm.sh @@ -45,6 +45,7 @@ function initialize_package() { log error "Failed to deploy package" exit 1 } + docker::deploy_config_importer $STACK "$COMPOSE_FILE_PATH/docker-compose.config.yml" "kafka-mapper-consumer-config-importer" "kafka-mapper-consumer" } function destroy_package() { diff --git a/mpi-mediator/docker-compose.yml b/mpi-mediator/docker-compose.yml index 084d7e39..9497ecf8 100644 --- a/mpi-mediator/docker-compose.yml +++ b/mpi-mediator/docker-compose.yml @@ -2,7 +2,7 @@ version: '3.9' services: mpi-mediator: - image: jembi/mpi-mediator:v2.1.1 + image: jembi/mpi-mediator:${MPI_MEDIATOR_VERSION} networks: openhim: kafka: diff --git a/mpi-mediator/importer/volume/openhim-import.json b/mpi-mediator/importer/volume/openhim-import.json index 5a1b11b9..cb40af5d 100644 --- a/mpi-mediator/importer/volume/openhim-import.json +++ b/mpi-mediator/importer/volume/openhim-import.json @@ -5,9 +5,7 @@ "surname": "User", "email": "root@openhim.org", "provider": "token", - "groups": [ - "admin" - ], + "groups": ["admin"], "passwordAlgorithm": "sha512", "passwordHash": "ea3824f17cf1379eb118a36bc7c8cf0f45712e2af7748567fca5313dec6fa66d61064e82a5e5cb88e998486ee3c7d0dac235bbeda8c341d6edc1c77406be2ab6", "passwordSalt": "d4f622c0404f09bd959bfb263efa3452", @@ -21,9 +19,7 @@ { "clientID": "test", "name": "Test Client", - "roles": [ - "instant" - ], + "roles": ["instant"], "customTokenID": "test" } ], @@ -98,7 +94,7 @@ "urlPattern": "^/fhir.*$", "methods": ["GET", "POST"], "type": "http", - "priority": null, + "priority": 2, "tcpPort": null, "tcpHost": null, "pollingSchedule": null, @@ -147,9 +143,7 @@ { "name": "MPI Orchestration for fhir bundles - Asynchronous flow", "urlPattern": "^/async/fhir/?$", - "methods": [ - "POST" - ], + "methods": ["POST"], "type": "http", "priority": null, "tcpPort": null, @@ -157,9 +151,7 @@ "pollingSchedule": null, "requestBody": true, "responseBody": true, - "allow": [ - "instant" - ], + "allow": ["instant"], "whitelist": [], "authType": "private", "routes": [ @@ -240,9 +232,7 @@ "pollingSchedule": null, "requestBody": true, "responseBody": true, - "allow": [ - "instant" - ], + "allow": ["instant"], "whitelist": [], "authType": "private", "routes": [ @@ -328,14 +318,9 @@ { "name": "MPI mediator", "urlPattern": "^(/async)?/fhir.*$", - "methods": [ - "POST", - "GET" - ], + "methods": ["POST", "GET"], "type": "http", - "allow": [ - "instant" - ], + "allow": ["instant"], "whitelist": [], "authType": "private", "routes": [ diff --git a/mpi-mediator/package-metadata.json b/mpi-mediator/package-metadata.json index 85c1f07e..ce30f0a9 100644 --- a/mpi-mediator/package-metadata.json +++ b/mpi-mediator/package-metadata.json @@ -7,6 +7,7 @@ "dependencies": ["fhir-datastore-hapi-fhir", "client-registry-jempi"], "environmentVariables": { "MPI_MEDIATOR_INSTANCES": 1, + "MPI_MEDIATOR_VERSION": "v2.3.0", "OPENHIM_MEDIATOR_URL": "https://openhim-core:8080", "TRUST_SELF_SIGNED": "true", "OPENHIM_USERNAME": "root@openhim.org", diff --git a/openhim-mapping-mediator/docker-compose.yml b/openhim-mapping-mediator/docker-compose.yml index cdfdd87b..3503fcf2 100644 --- a/openhim-mapping-mediator/docker-compose.yml +++ b/openhim-mapping-mediator/docker-compose.yml @@ -1,8 +1,8 @@ -version: '3.9' +version: "3.9" services: openhim-mapping-mediator: - image: jembi/openhim-mediator-mapping:v3.3.0 + image: ${OPENHIM_MAPPING_MEDIATOR_IMAGE} environment: OPENHIM_REGISTER: ${OPENHIM_REGISTER} MONGO_URL: ${OPENHIM_MONGO_URL} diff --git a/openhim-mapping-mediator/package-metadata.json b/openhim-mapping-mediator/package-metadata.json index 353515e5..f0b9cb0a 100644 --- a/openhim-mapping-mediator/package-metadata.json +++ b/openhim-mapping-mediator/package-metadata.json @@ -6,6 +6,7 @@ "version": "0.0.1", "dependencies": ["interoperability-layer-openhim"], "environmentVariables": { + "OPENHIM_MAPPING_MEDIATOR_IMAGE": "jembi/openhim-mediator-mapping:v3.3.0", "OPENHIM_URL": "https://openhim-core:8080", "OPENHIM_USERNAME": "root@openhim.org", "OPENHIM_MONGO_URL": "mongodb://mongo-1:27017/openhim", diff --git a/reprocess-mediator/docker-compose.dev.yml b/reprocess-mediator/docker-compose.dev.yml new file mode 100644 index 00000000..e376b2f1 --- /dev/null +++ b/reprocess-mediator/docker-compose.dev.yml @@ -0,0 +1,14 @@ +version: '3.9' + +services: + reprocess-mediator: + ports: + - target: 3000 + published: 3335 + mode: host + + reprocess-mediator-ui: + ports: + - target: 80 + published: 3030 + mode: host diff --git a/reprocess-mediator/docker-compose.yml b/reprocess-mediator/docker-compose.yml new file mode 100644 index 00000000..e1efc59a --- /dev/null +++ b/reprocess-mediator/docker-compose.yml @@ -0,0 +1,31 @@ +version: '3.9' + +services: + reprocess-mediator: + image: jembi/reprocess-mediator:${REPROCESS_CORE_VERSION} + networks: + openhim: + reprocess: + environment: + TRUST_SELF_SIGNED: ${TRUST_SELF_SIGNED} + OPENHIM_MEDIATOR_URL: ${OPENHIM_MEDIATOR_URL} + OPENHIM_USERNAME: ${OPENHIM_USERNAME} + OPENHIM_PASSWORD: ${OPENHIM_PASSWORD} + REGISTER_MEDIATOR: ${REGISTER_MEDIATOR} + + reprocess-mediator-ui: + image: jembi/reprocess-mediator-ui:${REPROCESS_UI_VERSION} + networks: + openhim: + reprocess: + environment: + REPROCESSOR_API_BASE_URL: ${REPROCESSOR_API_BASE_URL} + +networks: + openhim: + name: openhim_public + external: true + reprocess: + name: reprocess_public + external: true + diff --git a/reprocess-mediator/package-metadata.json b/reprocess-mediator/package-metadata.json new file mode 100644 index 00000000..97ca4643 --- /dev/null +++ b/reprocess-mediator/package-metadata.json @@ -0,0 +1,18 @@ +{ + "id": "reprocess-mediator", + "name": "Reprocess Mediator", + "description": "A mediator that allows for configuring and reprocessing of transactions", + "type": "use-case", + "version": "1.4.2", + "dependencies": ["interoperability-layer-openhim"], + "environmentVariables": { + "TRUST_SELF_SIGNED": "true", + "OPENHIM_MEDIATOR_URL": "https://openhim-core:8080", + "OPENHIM_USERNAME": "root@openhim.org", + "OPENHIM_PASSWORD": "instant101", + "REGISTER_MEDIATOR": "true", + "REPROCESSOR_API_BASE_URL": "http://reprocess-mediator:3000", + "REPROCESS_CORE_VERSION": "v0.1.0", + "REPROCESS_UI_VERSION": "v0.1.0" + } +} diff --git a/reprocess-mediator/swarm.sh b/reprocess-mediator/swarm.sh new file mode 100644 index 00000000..dcb182eb --- /dev/null +++ b/reprocess-mediator/swarm.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +declare ACTION="" +declare COMPOSE_FILE_PATH="" +declare UTILS_PATH="" +declare STACK="reprocess-mediator" +declare MODE="" + +function init_vars() { + ACTION=$1 + MODE=$2 + + COMPOSE_FILE_PATH=$( + cd "$(dirname "${BASH_SOURCE[0]}")" || exit + pwd -P + ) + + UTILS_PATH="${COMPOSE_FILE_PATH}/../utils" + + readonly ACTION + readonly COMPOSE_FILE_PATH + readonly UTILS_PATH + readonly STACK + readonly MODE +} + +# shellcheck disable=SC1091 +function import_sources() { + source "${UTILS_PATH}/docker-utils.sh" + source "${UTILS_PATH}/log.sh" +} + +function initialize_package() { + local reprocess_dev_compose_filename="" + + if [[ "${MODE}" == "dev" ]]; then + log info "Running package in DEV mode" + reprocess_dev_compose_filename="docker-compose.dev.yml" + else + log info "Running package in PROD mode" + fi + + ( + docker::deploy_service $STACK "${COMPOSE_FILE_PATH}" "docker-compose.yml" "$reprocess_dev_compose_filename" + ) || { + log error "Failed to deploy package" + exit 1 + } +} + +function destroy_package() { + docker::stack_destroy $STACK + + docker::prune_configs "reprocess-mediator" +} + +main() { + init_vars "$@" + import_sources + + if [[ "${ACTION}" == "init" ]] || [[ "${ACTION}" == "up" ]]; then + log info "Running package" + + initialize_package + elif [[ "${ACTION}" == "down" ]]; then + log info "Scaling down package" + + docker::scale_services $STACK 0 + elif [[ "${ACTION}" == "destroy" ]]; then + log info "Destroying package" + destroy_package + else + log error "Valid options are: init, up, down, or destroy" + fi +} + +main "$@" diff --git a/test/cucumber/features/cluster-mode/jempi.cluster.feature b/test/cucumber/features/cluster-mode/jempi.cluster.feature index 81279549..83210fd8 100644 --- a/test/cucumber/features/cluster-mode/jempi.cluster.feature +++ b/test/cucumber/features/cluster-mode/jempi.cluster.feature @@ -23,8 +23,6 @@ Feature: Client Registry JeMPI? And The service "jempi-alpha-02" should be started with 1 replica And The service "jempi-alpha-03" should be started with 1 replica And The service "jempi-zero-01" should be started with 1 replica - And The service "jempi-zero-02" should be started with 1 replica - And The service "jempi-zero-03" should be started with 1 replica And The service "jempi-async-receiver" should be started with 1 replica And The service "jempi-async-receiver" should be connected to the networks | kafka_public | jempi_default | @@ -41,12 +39,11 @@ Feature: Client Registry JeMPI? And The service "jempi-linker" should be started with 1 replica And The service "jempi-linker" should be connected to the networks | kafka_public | jempi_default | - And The service "jempi-api" should be started with 3 replica + And The service "jempi-api" should be started with 1 replica + And The service "mongo-1" should be started with 1 replica And The service "jempi-api" should be connected to the networks | kafka_public | jempi_default | And The service "jempi-postgresql-01" should be started with 1 replica - And The service "jempi-postgresql-02" should be started with 1 replica - And The service "jempi-postgresql-03" should be started with 1 replica And The service "jempi-web" should be started with 3 replica And The service "jempi-web" should be connected to the networks | reverse-proxy_public | keycloak_public | jempi_default | @@ -74,13 +71,9 @@ Feature: Client Registry JeMPI? And The service "jempi-em-calculator" should be removed And The service "jempi-linker" should be removed And The service "jempi-zero-01" should be removed - And The service "jempi-zero-02" should be removed - And The service "jempi-zero-03" should be removed And The service "jempi-api" should be removed And The service "jempi-web" should be removed And The service "jempi-postgresql-01" should be removed - And The service "jempi-postgresql-02" should be removed - And The service "jempi-postgresql-03" should be removed And The service "mongo-1" should be removed And The service "mongo-2" should be removed And The service "mongo-3" should be removed diff --git a/test/cucumber/features/single-mode/kafka-packages.feature b/test/cucumber/features/single-mode/kafka-packages.feature index c37b8f08..6b398d79 100644 --- a/test/cucumber/features/single-mode/kafka-packages.feature +++ b/test/cucumber/features/single-mode/kafka-packages.feature @@ -18,11 +18,11 @@ Feature: Kafka and its dependent packages? And There should be 1 volumes Scenario: Init Kafka Mapper Consumer - Given I use parameters "package init -n=kafka-mapper-consumer --only --dev --env-file=.env.local" + Given I use parameters "package init -n=interoperability-layer-openhim,kafka-mapper-consumer --only --dev --env-file=.env.local" When I launch the platform with params Then The service "kafka-mapper-consumer" should be started with 1 replica And The service "kafka-mapper-consumer" should be connected to the networks - | clickhouse_public | kafka_public | + | clickhouse_public | kafka_public | openhim_public | Scenario: Init Message Bus Kafka Given I use parameters "package init -n=kafka-unbundler-consumer --only --dev --env-file=.env.local" @@ -32,7 +32,7 @@ Feature: Kafka and its dependent packages? | kafka_public | Scenario: Destroy Kafka and its dependent packages - Given I use parameters "package destroy -n=kafka-mapper-consumer,kafka-unbundler-consumer --dev --env-file=.env.local" + Given I use parameters "package destroy -n=kafka-mapper-consumer,kafka-unbundler-consumer,interoperability-layer-openhim --dev --env-file=.env.local" When I launch the platform with params And The service "kafka-01" should be removed And The service "kafdrop" should be removed @@ -43,4 +43,4 @@ Feature: Kafka and its dependent packages? And There should be 0 volume And There should be 0 config And There should not be network - | kafka_public | clickhouse_public | prometheus_public | + | kafka_public | clickhouse_public | prometheus_public | openhim_public | diff --git a/test/cucumber/features/single-mode/recipe.feature b/test/cucumber/features/single-mode/recipe.feature index 54a93c50..c676be79 100644 --- a/test/cucumber/features/single-mode/recipe.feature +++ b/test/cucumber/features/single-mode/recipe.feature @@ -1,7 +1,7 @@ Feature: CDR-DW recipe? Does the recipe work as expected -Scenario: Init the CDR recipe + Scenario: Init the CDR recipe Given I use parameters "package init -p cdr-dw --dev --env-file=cdr-dw.env" When I launch the platform with params Then The service "mongo-1" should be started with 1 replica @@ -12,7 +12,6 @@ Scenario: Init the CDR recipe And The service "kafka-01" should be started with 1 replica And The service "kafdrop" should be started with 1 replica And The service "kafka-minion" should be started with 1 replica - And The service "keycloak-postgres-1" should be started with 1 replica And The service "identity-access-manager-keycloak" should be started with 1 replica And The service "jempi-ratel" should be started with 1 replica And The service "jempi-alpha-01" should be started with 1 replica @@ -35,27 +34,27 @@ Scenario: Init the CDR recipe And The service "dashboard-visualiser-superset" should be started with 1 replica And The service "analytics-datastore-clickhouse" should be started with 1 replica -Scenario: Send Fhir bundle and store the clinical data in the Fhir datastore, the patient info in the CR + Scenario: Send Fhir bundle and store the clinical data in the Fhir datastore, the patient info in the CR Given I have configured the cdr When I send a fhir patient bundle Then the clinical data should be stored in hapi fhir And the patient data in the Jempi client registry And the data should be stored in clickhouse -Scenario: Fetch International Patient summary (IPS) + Scenario: Fetch International Patient summary (IPS) When I then send a fhir patient summary request Then I should get a successful summary response -Scenario: Fetch everything for a patient (all the clinical data) + Scenario: Fetch everything for a patient (all the clinical data) When I then send a request for all the patient's clinical data Then I should get a successful everything response -Scenario: Bring down the servers + Scenario: Bring down the servers Given I use parameters "package down -p cdr-dw --env-file=cdr-dw.env" When I launch the platform with params Then a request to fetch data from the cdr should fail -Scenario: Bring up the servers and test + Scenario: Bring up the servers and test Given I use parameters "package up -p cdr-dw --dev --env-file=cdr-dw.env" When I launch the platform with params Then The service "mongo-1" should be started with 1 replica @@ -67,7 +66,7 @@ Scenario: Bring up the servers and test Then I should get a successful everything response And the data should be stored in clickhouse -Scenario: Destroy the services + Scenario: Destroy the services Given I use parameters "package remove -p cdr-dw --env-file=cdr-dw.env" When I launch the platform with params Then There should be 0 service diff --git a/test/cucumber/features/steps/recipesSteps.js b/test/cucumber/features/steps/recipesSteps.js index 9c119150..5785eaf4 100644 --- a/test/cucumber/features/steps/recipesSteps.js +++ b/test/cucumber/features/steps/recipesSteps.js @@ -1,16 +1,15 @@ -"use strict" +"use strict"; const axios = require("axios"); const fs = require("fs"); -const path = require('path'); +const path = require("path"); const chai = require("chai"); -const { ClickHouse } = require('clickhouse'); +const { ClickHouse } = require("clickhouse"); const { Given, When, Then, setDefaultTimeout } = require("@cucumber/cucumber"); setDefaultTimeout(30 * 60 * 1000); -const HOST = - process.env.HOST || 'localhost'; -const CLICKHOUSE_PORT = parseInt(process.env.CLICKHOUSE_PORT || '8124'); +const HOST = process.env.HOST || "localhost"; +const CLICKHOUSE_PORT = parseInt(process.env.CLICKHOUSE_PORT || "8124"); const CLICKHOUSE_DEBUG = Boolean(process.env.CLICKHOUSE_DEBUG || false); const { expect } = chai; @@ -22,69 +21,95 @@ const clickhouse = new ClickHouse({ raw: true, }); -const query = table => `SELECT * FROM ${table}`; +const query = (table) => `SELECT * FROM ${table}`; -const sendRequest = (url, method='POST', data={}) => { +const sendRequest = (url, method = "POST", data = {}) => { return axios({ url, headers: { - 'Content-Type': 'application/fhir+json', - Authorization: `Custom test` + "Content-Type": "application/fhir+json", + Authorization: `Custom test`, }, data, - method - }) + method, + }); }; let PatientID; Given("I have configured the cdr", async function () { await new Promise((resolve) => { - setTimeout(() => resolve(), 300000) + setTimeout(() => resolve(), 300000); }); const organization = JSON.parse( - fs.readFileSync(path.resolve(__dirname, '..' , 'resources', 'organization.json')) + fs.readFileSync( + path.resolve(__dirname, "..", "resources", "organization.json") + ) ); - this.cdrConfigResult = await sendRequest(`http://${HOST}:5001/fhir`, 'POST', organization); + this.cdrConfigResult = await sendRequest( + `http://${HOST}:5001/fhir`, + "POST", + organization + ); }); When("I send a fhir patient bundle", async function () { const fhirBundle = JSON.parse( - fs.readFileSync(path.resolve(__dirname, '..' , 'resources', 'fhirBundle.json')) + fs.readFileSync( + path.resolve(__dirname, "..", "resources", "fhirBundle.json") + ) ); - this.fhirBundleSentResult = await sendRequest(`http://${HOST}:5001/fhir`, 'POST', fhirBundle); + this.fhirBundleSentResult = await sendRequest( + `http://${HOST}:5001/fhir`, + "POST", + fhirBundle + ); }); When("I then send a fhir patient summary request", async function () { - this.IPSResult = await sendRequest(`http://${HOST}:5001/fhir/Patient/${PatientID}/$summary`, 'GET'); -}); - -When("I then send a request for all the patient's clinical data", async function () { - this.EverythingResult = await sendRequest(`http://${HOST}:5001/fhir/Patient/${PatientID}/$everything?_mdm=true`, 'GET'); + this.IPSResult = await sendRequest( + `http://${HOST}:5001/fhir/Patient/${PatientID}/$summary`, + "GET" + ); }); -When("I wait for the services to start up", async function() { +When( + "I then send a request for all the patient's clinical data", + async function () { + this.EverythingResult = await sendRequest( + `http://${HOST}:5001/fhir/Patient/${PatientID}/$everything?_mdm=true`, + "GET" + ); + } +); + +When("I wait for the services to start up", async function () { await new Promise((resolve) => { setTimeout(() => resolve(), 1500000); }); }); Then("the clinical data should be stored in hapi fhir", async function () { - this.fhirBundleSentResult.data.entry.forEach(rec => { + this.fhirBundleSentResult.data.entry.forEach((rec) => { expect(rec.response.status).to.match(/201|200/); }); }); Then("the patient data in the Jempi client registry", async function () { - const patientResource = this.fhirBundleSentResult.data.entry.filter(rec => rec.response.location.match('Patient'))[0]; + const patientResource = this.fhirBundleSentResult.data.entry.filter((rec) => + rec.response.location.match("Patient") + )[0]; - PatientID = patientResource.response.location.split('/')[1]; + PatientID = patientResource.response.location.split("/")[1]; - const patient = await sendRequest(`http://${HOST}:5001/fhir/links/Patient/${PatientID}`, 'GET'); + const patient = await sendRequest( + `http://${HOST}:5001/fhir/Patient/${PatientID}`, + "GET" + ); - expect(patient.data.link.filter(pat => pat.other.reference.match(`Patient/${PatientID}`)).length).to.equal(1); + expect(patient.data.id).to.be.equal(PatientID); }); Then("I should get a successful summary response", function () { @@ -96,22 +121,22 @@ Then("I should get a successful everything response", function () { }); Then("a request to fetch data from the cdr should fail", async function () { - await sendRequest(`http://${HOST}:5001/fhir/links/Patient/${PatientID}`).catch(err => { - expect(err.message).to.match(/ECONNREFUSED|socket hang up|ETIMEDOUT/); - }); + await sendRequest(`http://${HOST}:5001/fhir/Patient/${PatientID}`).catch( + (err) => { + expect(err.message).to.match(/ECONNREFUSED|socket hang up|ETIMEDOUT/); + } + ); }); Then("the data should be stored in clickhouse", async function () { await new Promise((resolve) => { - setTimeout(() => resolve(), 20000) + setTimeout(() => resolve(), 20000); }); - const patient = await clickhouse.query( - query("patient_example"), - ).toPromise(); - const observation = await clickhouse.query( - query("observation_example") - ).toPromise(); + const patient = await clickhouse.query(query("patient_example")).toPromise(); + const observation = await clickhouse + .query(query("observation_example")) + .toPromise(); expect(JSON.parse(patient).rows).to.be.greaterThan(0); expect(JSON.parse(observation).rows).to.be.greaterThan(0);