From e6eb5ecd9919fb2690796aa628360ffad7e7efb8 Mon Sep 17 00:00:00 2001 From: Brett Onions Date: Wed, 17 Apr 2024 09:10:11 +0200 Subject: [PATCH 01/43] Update docker-compose files and package metadata --- .../docker-compose-postgres.cluster.yml | 21 +++++++++---------- .../docker-compose-postgres.yml | 4 ++-- .../docker-compose.yml | 8 +++---- .../package-metadata.json | 8 +++++++ 4 files changed, 24 insertions(+), 17 deletions(-) diff --git a/identity-access-manager-keycloak/docker-compose-postgres.cluster.yml b/identity-access-manager-keycloak/docker-compose-postgres.cluster.yml index a6ec7061..4be183d9 100644 --- a/identity-access-manager-keycloak/docker-compose-postgres.cluster.yml +++ b/identity-access-manager-keycloak/docker-compose-postgres.cluster.yml @@ -1,4 +1,4 @@ -version: '3.9' +version: "3.9" services: keycloak-postgres-1: @@ -7,10 +7,10 @@ services: deploy: placement: constraints: - - "node.labels.name==node-1" + - "node.labels.name==${KEYCLOAK_POSTGRES_1}" keycloak-postgres-2: - image: bitnami/postgresql-repmgr:14 + image: ${POSTGRES_IMAGE} environment: POSTGRESQL_PASSWORD: ${KC_POSTGRESQL_PASSWORD} POSTGRESQL_USERNAME: ${KC_POSTGRESQL_USERNAME} @@ -22,12 +22,12 @@ services: REPMGR_PRIMARY_HOST: ${KC_REPMGR_PRIMARY_HOST} REPMGR_PARTNER_NODES: ${KC_REPMGR_PARTNER_NODES} volumes: - - 'keycloak-postgres-2-data:/bitnami/postgresql' + - "keycloak-postgres-2-data:/bitnami/postgresql" deploy: placement: constraints: - - "node.labels.name==node-2" - replicas: 1 + - "node.labels.name==${KEYCLOAK_POSTGRES_2}" + replicas: ${POSTGRES_REPLICAS} resources: limits: cpus: ${KC_POSTGRES_CPU_LIMIT} @@ -39,9 +39,8 @@ services: default: keycloak_backup_net: {} - keycloak-postgres-3: - image: bitnami/postgresql-repmgr:14 + image: ${POSTGRES_IMAGE} environment: POSTGRESQL_PASSWORD: ${KC_POSTGRESQL_PASSWORD} POSTGRESQL_USERNAME: ${KC_POSTGRESQL_USERNAME} @@ -53,12 +52,12 @@ services: REPMGR_PRIMARY_HOST: ${KC_REPMGR_PRIMARY_HOST} REPMGR_PARTNER_NODES: ${KC_REPMGR_PARTNER_NODES} volumes: - - 'keycloak-postgres-3-data:/bitnami/postgresql' + - "keycloak-postgres-3-data:/bitnami/postgresql" deploy: placement: constraints: - - "node.labels.name==node-3" - replicas: 1 + - "node.labels.name==${KEYCLOAK_POSTGRES_3}" + replicas: ${POSTGRES_REPLICAS} resources: limits: cpus: ${KC_POSTGRES_CPU_LIMIT} diff --git a/identity-access-manager-keycloak/docker-compose-postgres.yml b/identity-access-manager-keycloak/docker-compose-postgres.yml index 4a8f2279..4fa41d41 100644 --- a/identity-access-manager-keycloak/docker-compose-postgres.yml +++ b/identity-access-manager-keycloak/docker-compose-postgres.yml @@ -2,7 +2,7 @@ version: "3.9" services: keycloak-postgres-1: - image: bitnami/postgresql-repmgr:14 + image: ${POSTGRES_IMAGE} environment: POSTGRESQL_PASSWORD: ${KC_POSTGRESQL_PASSWORD} POSTGRESQL_USERNAME: ${KC_POSTGRESQL_USERNAME} @@ -17,7 +17,7 @@ services: volumes: - "keycloak-postgres-1-data:/bitnami/postgresql" deploy: - replicas: 1 + replicas: ${POSTGRES_REPLICAS} resources: limits: cpus: ${KC_POSTGRES_CPU_LIMIT} diff --git a/identity-access-manager-keycloak/docker-compose.yml b/identity-access-manager-keycloak/docker-compose.yml index fc5e4c28..22589738 100644 --- a/identity-access-manager-keycloak/docker-compose.yml +++ b/identity-access-manager-keycloak/docker-compose.yml @@ -1,14 +1,14 @@ -version: '3.9' +version: "3.9" services: identity-access-manager-keycloak: - image: keycloak/keycloak:20.0 + image: ${KEYCLOAK_IMAGE} command: [ "start", "--proxy=edge", "--hostname-url=${KC_FRONTEND_URL}", - "--import-realm" + "--import-realm", ] hostname: identity-access-manager-keycloak healthcheck: @@ -44,7 +44,7 @@ services: KC_OPENHIM_ROOT_URL: ${KC_OPENHIM_ROOT_URL} deploy: placement: - max_replicas_per_node: 1 + max_replicas_per_node: ${KEYCLOAK_MAX_REPLICAS_PER_NODE} networks: reverse-proxy: public: diff --git a/identity-access-manager-keycloak/package-metadata.json b/identity-access-manager-keycloak/package-metadata.json index 928ca89a..44071228 100644 --- a/identity-access-manager-keycloak/package-metadata.json +++ b/identity-access-manager-keycloak/package-metadata.json @@ -6,6 +6,14 @@ "version": "0.0.1", "dependencies": [], "environmentVariables": { + "POSTGRES_IMAGE": "bitnami/postgresql-repmgr:14", + "KEYCLOAK_POSTGRES_1": "node-1", + "KEYCLOAK_POSTGRES_2": "node-2", + "KEYCLOAK_POSTGRES_3": "node-3", + "POSTGRES_REPLICAS": "1", + "KEYCLOAK_IMAGE": "keycloak/keycloak:20.0", + "KEYCLOAK_REPLICAS": "1", + "KEYCLOAK_MAX_REPLICAS_PER_NODE": "1", "KEYCLOAK_ADMIN": "admin", "KEYCLOAK_ADMIN_PASSWORD": "dev_password_only", "KC_FRONTEND_URL": "http://localhost:9088", From 4340b7eba315083d7c0f8c36d3c5744aadc24739 Mon Sep 17 00:00:00 2001 From: Brett Onions Date: Thu, 18 Apr 2024 08:46:45 +0200 Subject: [PATCH 02/43] Update Docker Compose files and package metadata --- .../docker-compose-mongo.cluster.yml | 26 +++++++++---------- .../docker-compose-mongo.yml | 4 +-- .../docker-compose.await-helper-mongo.yml | 6 ++--- .../docker-compose.await-helper.yml | 6 ++--- .../docker-compose.yml | 10 +++---- .../package-metadata.json | 9 +++++++ 6 files changed, 35 insertions(+), 26 deletions(-) diff --git a/interoperability-layer-openhim/docker-compose-mongo.cluster.yml b/interoperability-layer-openhim/docker-compose-mongo.cluster.yml index 566ec6b5..79868fdf 100644 --- a/interoperability-layer-openhim/docker-compose-mongo.cluster.yml +++ b/interoperability-layer-openhim/docker-compose-mongo.cluster.yml @@ -1,23 +1,23 @@ -version: '3.9' +version: "3.9" services: mongo-1: - command: ['--replSet', 'mongo-set', '--wiredTigerCacheSizeGB', '0.5'] + command: ["--replSet", "mongo-set", "--wiredTigerCacheSizeGB", "0.5"] deploy: placement: constraints: - - "node.labels.name==node-1" + - "node.labels.name==${MONGO_1_PLACEMENT}" mongo-2: - image: mongo:4.2 + image: ${MONGO_IMAGE} volumes: - - 'openhim-mongo-02:/data/db' - - 'openhim-mongo-02-config:/data/configdb' - command: ['--replSet', 'mongo-set', '--wiredTigerCacheSizeGB', '0.5'] + - "openhim-mongo-02:/data/db" + - "openhim-mongo-02-config:/data/configdb" + command: ["--replSet", "mongo-set", "--wiredTigerCacheSizeGB", "0.5"] deploy: placement: constraints: - - "node.labels.name==node-2" + - "node.labels.name==${MONGO_2_PLACEMENT}" replicas: 1 resources: limits: @@ -32,15 +32,15 @@ services: mongo_backup_net: mongo-3: - image: mongo:4.2 + image: ${MONGO_IMAGE} volumes: - - 'openhim-mongo-03:/data/db' - - 'openhim-mongo-03-config:/data/configdb' - command: ['--replSet', 'mongo-set', '--wiredTigerCacheSizeGB', '0.5'] + - "openhim-mongo-03:/data/db" + - "openhim-mongo-03-config:/data/configdb" + command: ["--replSet", "mongo-set", "--wiredTigerCacheSizeGB", "0.5"] deploy: placement: constraints: - - "node.labels.name==node-3" + - "node.labels.name==${MONGO_3_PLACEMENT}" replicas: 1 resources: limits: diff --git a/interoperability-layer-openhim/docker-compose-mongo.yml b/interoperability-layer-openhim/docker-compose-mongo.yml index c67589f6..69117ee7 100644 --- a/interoperability-layer-openhim/docker-compose-mongo.yml +++ b/interoperability-layer-openhim/docker-compose-mongo.yml @@ -2,8 +2,8 @@ version: "3.9" services: mongo-1: - image: mongo:4.2 - command: ['--replSet', 'mongo-set', '--wiredTigerCacheSizeGB', '0.5'] + image: ${MONGO_IMAGE} + command: ["--replSet", "mongo-set", "--wiredTigerCacheSizeGB", "0.5"] volumes: - "openhim-mongo-01:/data/db" - "openhim-mongo-01-config:/data/configdb" diff --git a/interoperability-layer-openhim/docker-compose.await-helper-mongo.yml b/interoperability-layer-openhim/docker-compose.await-helper-mongo.yml index c02d4ffe..82729d17 100644 --- a/interoperability-layer-openhim/docker-compose.await-helper-mongo.yml +++ b/interoperability-layer-openhim/docker-compose.await-helper-mongo.yml @@ -1,10 +1,10 @@ -version: '3.9' +version: "3.9" services: await-helper: - image: jembi/await-helper:1.0.1 + image: ${AWAIT_HELPER_IMAGE} deploy: replicas: 1 restart_policy: condition: none - command: '-k http://mongo-1:27017' + command: "-k http://mongo-1:27017" diff --git a/interoperability-layer-openhim/docker-compose.await-helper.yml b/interoperability-layer-openhim/docker-compose.await-helper.yml index 8fe4c43e..9625820b 100644 --- a/interoperability-layer-openhim/docker-compose.await-helper.yml +++ b/interoperability-layer-openhim/docker-compose.await-helper.yml @@ -1,10 +1,10 @@ -version: '3.9' +version: "3.9" services: await-helper: - image: jembi/await-helper:1.0.1 + image: ${AWAIT_HELPER_IMAGE} deploy: replicas: 1 restart_policy: condition: none - command: '-k https://openhim-core:8080/heartbeat' + command: "-k https://openhim-core:8080/heartbeat" diff --git a/interoperability-layer-openhim/docker-compose.yml b/interoperability-layer-openhim/docker-compose.yml index 2a2073f3..778cc5e2 100644 --- a/interoperability-layer-openhim/docker-compose.yml +++ b/interoperability-layer-openhim/docker-compose.yml @@ -1,8 +1,8 @@ -version: '3.9' +version: "3.9" services: openhim-core: - image: jembi/openhim-core:v8.4.0 + image: ${OPENHIM_CORE_IMAGE} networks: kafka: hapi-fhir: @@ -25,7 +25,7 @@ services: deploy: replicas: ${OPENHIM_CORE_INSTANCES} placement: - max_replicas_per_node: 1 + max_replicas_per_node: ${OPENHIM_CORE_MAX_REPLICAS_PER_NODE} resources: limits: cpus: ${OPENHIM_CPU_LIMIT} @@ -39,7 +39,7 @@ services: - prometheus-port=8080 openhim-console: - image: jembi/openhim-console:v1.19.0 + image: ${OPENHIM_CONSOLE_IMAGE} environment: OPENHIM_CORE_MEDIATOR_HOSTNAME: ${OPENHIM_CORE_MEDIATOR_HOSTNAME} OPENHIM_MEDIATOR_API_PORT: ${OPENHIM_MEDIATOR_API_PORT} @@ -56,7 +56,7 @@ services: deploy: replicas: ${OPENHIM_CONSOLE_INSTANCES} placement: - max_replicas_per_node: 1 + max_replicas_per_node: ${OPENHIM_CONSOLE_MAX_REPLICAS_PER_NODE} resources: limits: cpus: ${OPENHIM_CONSOLE_CPU_LIMIT} diff --git a/interoperability-layer-openhim/package-metadata.json b/interoperability-layer-openhim/package-metadata.json index 3158f12f..09026554 100644 --- a/interoperability-layer-openhim/package-metadata.json +++ b/interoperability-layer-openhim/package-metadata.json @@ -6,9 +6,18 @@ "type": "infrastructure", "dependencies": [], "environmentVariables": { + "OPENHIM_CORE_IMAGE": "jembi/openhim-core:v8.4.0", + "OPENHIM_CONSOLE_IMAGE": "jembi/openhim-console:v1.19.0", + "MONGO_IMAGE": "mongo:4.2", + "AWAIT_HELPER_IMAGE": "jembi/await-helper:1.0.1", + "MONGO_1_PLACEMENT": "node-1", + "MONGO_2_PLACEMENT": "node-2", + "MONGO_3_PLACEMENT": "node-3", "MONGO_SET_COUNT": "1", "OPENHIM_CORE_INSTANCES": "1", + "OPENHIM_CORE_MAX_REPLICAS_PER_NODE": "1", "OPENHIM_CONSOLE_INSTANCES": "1", + "OPENHIM_CONSOLE_MAX_REPLICAS_PER_NODE": "1", "OPENHIM_CORE_MEDIATOR_HOSTNAME": "localhost", "OPENHIM_MEDIATOR_API_PORT": "8080", "OPENHIM_CPU_LIMIT": "0", From 6f3de82fabfa7a132d4f274c3736d507aef5f4f3 Mon Sep 17 00:00:00 2001 From: Brett Onions Date: Thu, 18 Apr 2024 08:48:50 +0200 Subject: [PATCH 03/43] Update job-scheduler-ofelia image and add JOB_SCHEDULER_OFELIA_IMAGE environment variable --- job-scheduler-ofelia/docker-compose.yml | 6 +++--- job-scheduler-ofelia/package-metadata.json | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/job-scheduler-ofelia/docker-compose.yml b/job-scheduler-ofelia/docker-compose.yml index 5cd0e3c8..00740f61 100644 --- a/job-scheduler-ofelia/docker-compose.yml +++ b/job-scheduler-ofelia/docker-compose.yml @@ -1,8 +1,8 @@ -version: '3.9' +version: "3.9" services: job-scheduler-ofelia: - image: mcuadros/ofelia:v0.3.6 + image: ${JOB_SCHEDULER_OFELIA_IMAGE} command: daemon --config=/tmp/config.ini configs: - target: /tmp/config.ini @@ -14,5 +14,5 @@ configs: ofelia-config.ini: file: ./config.ini name: ofelia-config.ini-${ofelia_config_ini_DIGEST:?err} - labels: + labels: name: ofelia diff --git a/job-scheduler-ofelia/package-metadata.json b/job-scheduler-ofelia/package-metadata.json index a4592a6d..719dffae 100644 --- a/job-scheduler-ofelia/package-metadata.json +++ b/job-scheduler-ofelia/package-metadata.json @@ -6,6 +6,7 @@ "version": "0.0.1", "dependencies": [], "environmentVariables": { + "JOB_SCHEDULER_OFELIA_IMAGE": "mcuadros/ofelia:v0.3.6", "RENEWAL_EMAIL": "dummy@jembi.org", "STAGING": "true", "DOMAIN_NAME": "localhost", From 57bf3609916380072833d920b7484cac70debab1 Mon Sep 17 00:00:00 2001 From: Brett Onions Date: Thu, 18 Apr 2024 10:52:32 +0200 Subject: [PATCH 04/43] Update docker-compose.yml and package-metadata.json --- openhim-mapping-mediator/docker-compose.yml | 4 ++-- openhim-mapping-mediator/package-metadata.json | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/openhim-mapping-mediator/docker-compose.yml b/openhim-mapping-mediator/docker-compose.yml index cdfdd87b..3503fcf2 100644 --- a/openhim-mapping-mediator/docker-compose.yml +++ b/openhim-mapping-mediator/docker-compose.yml @@ -1,8 +1,8 @@ -version: '3.9' +version: "3.9" services: openhim-mapping-mediator: - image: jembi/openhim-mediator-mapping:v3.3.0 + image: ${OPENHIM_MAPPING_MEDIATOR_IMAGE} environment: OPENHIM_REGISTER: ${OPENHIM_REGISTER} MONGO_URL: ${OPENHIM_MONGO_URL} diff --git a/openhim-mapping-mediator/package-metadata.json b/openhim-mapping-mediator/package-metadata.json index 353515e5..f0b9cb0a 100644 --- a/openhim-mapping-mediator/package-metadata.json +++ b/openhim-mapping-mediator/package-metadata.json @@ -6,6 +6,7 @@ "version": "0.0.1", "dependencies": ["interoperability-layer-openhim"], "environmentVariables": { + "OPENHIM_MAPPING_MEDIATOR_IMAGE": "jembi/openhim-mediator-mapping:v3.3.0", "OPENHIM_URL": "https://openhim-core:8080", "OPENHIM_USERNAME": "root@openhim.org", "OPENHIM_MONGO_URL": "mongodb://mongo-1:27017/openhim", From 104aec870850bd349633458b57532346dca8829e Mon Sep 17 00:00:00 2001 From: ItsMurumba Date: Mon, 29 Apr 2024 10:11:38 +0300 Subject: [PATCH 05/43] Adds Reprocess package (UI + Mediator) --- config.yaml | 8 +++ reprocess-mediator/docker-compose.dev.yml | 14 +++++ reprocess-mediator/docker-compose.yml | 34 ++++++++++ reprocess-mediator/package-metadata.json | 15 +++++ reprocess-mediator/swarm.sh | 75 +++++++++++++++++++++++ 5 files changed, 146 insertions(+) create mode 100644 reprocess-mediator/docker-compose.dev.yml create mode 100644 reprocess-mediator/docker-compose.yml create mode 100644 reprocess-mediator/package-metadata.json create mode 100644 reprocess-mediator/swarm.sh diff --git a/config.yaml b/config.yaml index c548503f..041ed6fb 100644 --- a/config.yaml +++ b/config.yaml @@ -23,6 +23,7 @@ packages: - client-registry-jempi - identity-access-manager-keycloak - openhim-mapping-mediator + - reprocess-mediator profiles: - name: cdr-dw @@ -72,3 +73,10 @@ profiles: - openhim-mapping-mediator envFiles: - mpi.env + + - name: reprocess + packages: + - reprocess-mediator + - reverse-proxy-nginx + envFiles: + - .env diff --git a/reprocess-mediator/docker-compose.dev.yml b/reprocess-mediator/docker-compose.dev.yml new file mode 100644 index 00000000..ee40381a --- /dev/null +++ b/reprocess-mediator/docker-compose.dev.yml @@ -0,0 +1,14 @@ +version: '3.9' + +services: + reprocess-mediator: + ports: + - target: 3000 + published: 3000 + mode: host + + reprocess-mediator-ui: + ports: + - target: 80 + published: 3030 + mode: host diff --git a/reprocess-mediator/docker-compose.yml b/reprocess-mediator/docker-compose.yml new file mode 100644 index 00000000..93ac9cfd --- /dev/null +++ b/reprocess-mediator/docker-compose.yml @@ -0,0 +1,34 @@ +version: '3.9' + +services: + reprocess-mediator: + image: reprocess-mediator:latest + networks: + openhim: + reprocess: + environment: + TRUST_SELF_SIGNED: ${TRUST_SELF_SIGNED} + OPENHIM_API_URL: ${OPENHIM_API_URL} + OPENHIM_USERNAME: ${OPENHIM_USERNAME} + OPENHIM_PASSWORD: ${OPENHIM_PASSWORD} + REGISTER_MEDIATOR: ${REGISTER_MEDIATOR} + + reprocess-mediator-ui: + image: reprocess-mediator-ui:latest + networks: + reprocess: + reprocessui: + environment: + REPROCESSOR_API_BASE_URL: ${REPROCESSOR_API_BASE_URL} + +networks: + openhim: + name: openhim_public + external: true + reprocess: + name: reprocess_public + external: true + reprocessui: + name: reprocess_ui_public + external: true + diff --git a/reprocess-mediator/package-metadata.json b/reprocess-mediator/package-metadata.json new file mode 100644 index 00000000..25d9876e --- /dev/null +++ b/reprocess-mediator/package-metadata.json @@ -0,0 +1,15 @@ +{ + "id": "reprocess-mediator", + "name": "Reprocess Mediator", + "description": "A mediator that allows for configuring and reprocess of transactions", + "type": "use-case", + "version": "1.4.2", + "dependencies": ["interoperability-layer-openhim"], + "environmentVariables": { + "TRUST_SELF_SIGNED": "true", + "OPENHIM_API_URL": "https://openhim-core:8080", + "OPENHIM_USERNAME": "root@openhim.org", + "OPENHIM_PASSWORD": "instant101", + "REGISTER_MEDIATOR": "true" + } + } diff --git a/reprocess-mediator/swarm.sh b/reprocess-mediator/swarm.sh new file mode 100644 index 00000000..fc8924b5 --- /dev/null +++ b/reprocess-mediator/swarm.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +declare ACTION="" +declare COMPOSE_FILE_PATH="" +declare UTILS_PATH="" +declare STACK="reprocess-mediator" +declare MODE="" + +function init_vars() { + ACTION=$1 + + COMPOSE_FILE_PATH=$( + cd "$(dirname "${BASH_SOURCE[0]}")" || exit + pwd -P + ) + + UTILS_PATH="${COMPOSE_FILE_PATH}/../utils" + + readonly ACTION + readonly COMPOSE_FILE_PATH + readonly UTILS_PATH + readonly STACK +} + +# shellcheck disable=SC1091 +function import_sources() { + source "${UTILS_PATH}/docker-utils.sh" + source "${UTILS_PATH}/log.sh" +} + +function initialize_package() { + local reprocess_dev_compose_filename="" + + if [[ "${MODE}" == "dev" ]]; then + log info "Running package in DEV mode" + reprocess_dev_compose_filename="docker-compose.dev.yml" + else + log info "Running package in PROD mode" + fi + + ( + docker::deploy_service $STACK "${COMPOSE_FILE_PATH}" "docker-compose.yml" "$reprocess_dev_compose_filename" + ) || { + log error "Failed to deploy package" + exit 1 + } +} + +function destroy_package() { + docker::stack_destroy $STACK + + docker::prune_configs "reprocess-mediator" +} + +main() { + init_vars "$@" + import_sources + + if [[ "${ACTION}" == "init" ]] || [[ "${ACTION}" == "up" ]]; then + log info "Running package" + + initialize_package + elif [[ "${ACTION}" == "down" ]]; then + log info "Scaling down package" + + docker::scale_services $STACK 0 + elif [[ "${ACTION}" == "destroy" ]]; then + log info "Destroying package" + destroy_package + else + log error "Valid options are: init, up, down, or destroy" + fi +} + +main "$@" From d0cac8f8eebb19e79e3ac7df9362f07e6933f14a Mon Sep 17 00:00:00 2001 From: ItsMurumba Date: Tue, 30 Apr 2024 19:54:32 +0300 Subject: [PATCH 06/43] Adds working reprocess mediator and ui --- config.yaml | 1 + interoperability-layer-openhim/docker-compose.yml | 5 +++++ interoperability-layer-openhim/package-metadata.json | 3 ++- reprocess-mediator/docker-compose.yml | 11 ++++------- reprocess-mediator/package-metadata.json | 7 +++++-- reprocess-mediator/swarm.sh | 2 ++ 6 files changed, 19 insertions(+), 10 deletions(-) diff --git a/config.yaml b/config.yaml index 041ed6fb..05894f1f 100644 --- a/config.yaml +++ b/config.yaml @@ -80,3 +80,4 @@ profiles: - reverse-proxy-nginx envFiles: - .env + dev: true diff --git a/interoperability-layer-openhim/docker-compose.yml b/interoperability-layer-openhim/docker-compose.yml index 2a2073f3..60cc7a09 100644 --- a/interoperability-layer-openhim/docker-compose.yml +++ b/interoperability-layer-openhim/docker-compose.yml @@ -11,6 +11,7 @@ services: public: default: prometheus: + reprocess: environment: - mongo_url=${OPENHIM_MONGO_URL} - mongo_atnaUrl=${OPENHIM_MONGO_ATNAURL} @@ -22,6 +23,7 @@ services: - api_openid_callbackUrl=${KC_OPENHIM_ROOT_URL} - api_openid_clientId=${KC_OPENHIM_CLIENT_ID} - api_openid_clientSecret=${KC_OPENHIM_CLIENT_SECRET} + - openhimConsoleBaseUrl=${OPENHIM_CONSOLE_BASE_URL} deploy: replicas: ${OPENHIM_CORE_INSTANCES} placement: @@ -84,4 +86,7 @@ networks: prometheus: name: prometheus_public external: true + reprocess: + name: reprocess_public + external: true default: diff --git a/interoperability-layer-openhim/package-metadata.json b/interoperability-layer-openhim/package-metadata.json index 3158f12f..59107f32 100644 --- a/interoperability-layer-openhim/package-metadata.json +++ b/interoperability-layer-openhim/package-metadata.json @@ -33,6 +33,7 @@ "KC_OPENHIM_CLIENT_ID": "openhim-oauth", "KC_OPENHIM_CLIENT_SECRET": "tZKfEbWf0Ka5HBNZwFrdSyQH2xT1sNMR", "KC_OPENHIM_ROOT_URL": "http://localhost:9000", - "KC_API_URL": "http://identity-access-manager-keycloak:8080" + "KC_API_URL": "http://identity-access-manager-keycloak:8080", + "OPENHIM_CONSOLE_BASE_URL": "http://localhost:9000" } } diff --git a/reprocess-mediator/docker-compose.yml b/reprocess-mediator/docker-compose.yml index 93ac9cfd..e1efc59a 100644 --- a/reprocess-mediator/docker-compose.yml +++ b/reprocess-mediator/docker-compose.yml @@ -2,22 +2,22 @@ version: '3.9' services: reprocess-mediator: - image: reprocess-mediator:latest + image: jembi/reprocess-mediator:${REPROCESS_CORE_VERSION} networks: openhim: reprocess: environment: TRUST_SELF_SIGNED: ${TRUST_SELF_SIGNED} - OPENHIM_API_URL: ${OPENHIM_API_URL} + OPENHIM_MEDIATOR_URL: ${OPENHIM_MEDIATOR_URL} OPENHIM_USERNAME: ${OPENHIM_USERNAME} OPENHIM_PASSWORD: ${OPENHIM_PASSWORD} REGISTER_MEDIATOR: ${REGISTER_MEDIATOR} reprocess-mediator-ui: - image: reprocess-mediator-ui:latest + image: jembi/reprocess-mediator-ui:${REPROCESS_UI_VERSION} networks: + openhim: reprocess: - reprocessui: environment: REPROCESSOR_API_BASE_URL: ${REPROCESSOR_API_BASE_URL} @@ -28,7 +28,4 @@ networks: reprocess: name: reprocess_public external: true - reprocessui: - name: reprocess_ui_public - external: true diff --git a/reprocess-mediator/package-metadata.json b/reprocess-mediator/package-metadata.json index 25d9876e..b8ae6de8 100644 --- a/reprocess-mediator/package-metadata.json +++ b/reprocess-mediator/package-metadata.json @@ -7,9 +7,12 @@ "dependencies": ["interoperability-layer-openhim"], "environmentVariables": { "TRUST_SELF_SIGNED": "true", - "OPENHIM_API_URL": "https://openhim-core:8080", + "OPENHIM_MEDIATOR_URL": "https://openhim-core:8080", "OPENHIM_USERNAME": "root@openhim.org", "OPENHIM_PASSWORD": "instant101", - "REGISTER_MEDIATOR": "true" + "REGISTER_MEDIATOR": "true", + "REPROCESSOR_API_BASE_URL": "http://reprocess-mediator:3000", + "REPROCESS_CORE_VERSION": "latest", + "REPROCESS_UI_VERSION": "latest" } } diff --git a/reprocess-mediator/swarm.sh b/reprocess-mediator/swarm.sh index fc8924b5..dcb182eb 100644 --- a/reprocess-mediator/swarm.sh +++ b/reprocess-mediator/swarm.sh @@ -8,6 +8,7 @@ declare MODE="" function init_vars() { ACTION=$1 + MODE=$2 COMPOSE_FILE_PATH=$( cd "$(dirname "${BASH_SOURCE[0]}")" || exit @@ -20,6 +21,7 @@ function init_vars() { readonly COMPOSE_FILE_PATH readonly UTILS_PATH readonly STACK + readonly MODE } # shellcheck disable=SC1091 From 2c3d74d2265bae63a612f9aa4270882961695528 Mon Sep 17 00:00:00 2001 From: ItsMurumba Date: Mon, 6 May 2024 10:13:09 +0300 Subject: [PATCH 07/43] Implement suggestions and code cleanup --- config.yaml | 7 ------- reprocess-mediator/package-metadata.json | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/config.yaml b/config.yaml index 55b7293e..784faeac 100644 --- a/config.yaml +++ b/config.yaml @@ -75,10 +75,3 @@ profiles: envFiles: - mpi.env - - name: reprocess - packages: - - reprocess-mediator - - reverse-proxy-nginx - envFiles: - - .env - dev: true diff --git a/reprocess-mediator/package-metadata.json b/reprocess-mediator/package-metadata.json index b8ae6de8..4393bc6e 100644 --- a/reprocess-mediator/package-metadata.json +++ b/reprocess-mediator/package-metadata.json @@ -1,7 +1,7 @@ { "id": "reprocess-mediator", "name": "Reprocess Mediator", - "description": "A mediator that allows for configuring and reprocess of transactions", + "description": "A mediator that allows for configuring and reprocessing of transactions", "type": "use-case", "version": "1.4.2", "dependencies": ["interoperability-layer-openhim"], From 8c73d8eb80318eb4968c1284530f57965627830b Mon Sep 17 00:00:00 2001 From: ItsMurumba Date: Mon, 6 May 2024 10:15:19 +0300 Subject: [PATCH 08/43] Spelling correction on docs --- documentation/recipes/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/recipes/README.md b/documentation/recipes/README.md index 5382f24f..9e792ad2 100644 --- a/documentation/recipes/README.md +++ b/documentation/recipes/README.md @@ -4,7 +4,7 @@ description: Pre-defined recipes for common use cases # 📜 Recipes -OpenHIM platform comes bundles with a set of generic packages that can be deployed and configured to support a number of different use cases. To help users of OpenHIM Platform get started with something they can make use of immediately, a number of default OpenHIM Platform reciepes are provided. These help you get started with everything you need setup and configured for a particular use case. +OpenHIM platform comes bundles with a set of generic packages that can be deployed and configured to support a number of different use cases. To help users of OpenHIM Platform get started with something they can make use of immediately, a number of default OpenHIM Platform recipes are provided. These help you get started with everything you need setup and configured for a particular use case. We current support the following default recipes: From 0835c33274f91631fd775bbb17c9da34f5150272 Mon Sep 17 00:00:00 2001 From: Ryan Crichton Date: Wed, 8 May 2024 11:56:31 +0200 Subject: [PATCH 09/43] Add multiplatform architecture build to github action --- .github/workflows/docker-publish.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index d42c484d..36fc0451 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -11,25 +11,33 @@ jobs: build-and-push: runs-on: ubuntu-20.04 steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build and push tag if: ${{ github.ref_name != 'main' }} - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v5 with: + platforms: linux/amd64,linux/arm64 push: true tags: jembi/platform:${{ github.ref_name }} - name: Build and push latest if: ${{ github.ref_name == 'main' }} - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v5 with: + platforms: linux/amd64,linux/arm64 push: true tags: jembi/platform:latest From e4e4780b3e5767d59841951cf9bf0e6afeb7540d Mon Sep 17 00:00:00 2001 From: Brett Onions Date: Wed, 8 May 2024 12:28:18 +0200 Subject: [PATCH 10/43] chore: update OPENHIM_CONSOLE_IMAGE to v1.18.2 in package-metadata.json --- interoperability-layer-openhim/package-metadata.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interoperability-layer-openhim/package-metadata.json b/interoperability-layer-openhim/package-metadata.json index e64e7bee..6825d8b4 100644 --- a/interoperability-layer-openhim/package-metadata.json +++ b/interoperability-layer-openhim/package-metadata.json @@ -7,7 +7,7 @@ "dependencies": [], "environmentVariables": { "OPENHIM_CORE_IMAGE": "jembi/openhim-core:v8.4.0", - "OPENHIM_CONSOLE_IMAGE": "jembi/openhim-console:v1.19.0", + "OPENHIM_CONSOLE_IMAGE": "jembi/openhim-console:v1.18.2", "MONGO_IMAGE": "mongo:4.2", "AWAIT_HELPER_IMAGE": "jembi/await-helper:1.0.1", "MONGO_1_PLACEMENT": "node-1", From 072e671ba427142e0d1a0da2abb47ccd56880d14 Mon Sep 17 00:00:00 2001 From: Brett Onions Date: Wed, 8 May 2024 13:33:11 +0200 Subject: [PATCH 11/43] responding to comments --- interoperability-layer-openhim/package-metadata.json | 1 - 1 file changed, 1 deletion(-) diff --git a/interoperability-layer-openhim/package-metadata.json b/interoperability-layer-openhim/package-metadata.json index 1f22c923..05bfa2f3 100644 --- a/interoperability-layer-openhim/package-metadata.json +++ b/interoperability-layer-openhim/package-metadata.json @@ -35,7 +35,6 @@ "OPENHIM_MONGO_MEMORY_RESERVE": "500M", "OPENHIM_MONGO_URL": "mongodb://mongo-1:27017/openhim", "OPENHIM_MONGO_ATNAURL": "mongodb://mongo-1:27017/openhim", - "OPENHIM_CONSOLE_VERSION": "jembi/openhim-console:microfrontend-poc", "KAFKA_HOSTS": "kafka-01:9092", "KC_REALM_NAME": "platform-realm", "KC_FRONTEND_URL": "http://localhost:9088", From 79e8d1d7718a63c7ba069e8ba1f23620b0c97d9e Mon Sep 17 00:00:00 2001 From: drono Date: Wed, 8 May 2024 14:18:58 +0300 Subject: [PATCH 12/43] CU-86bynyk8j - Register Platform Default Portal applications During init --- config.yaml | 10 ++- .../importer/docker-compose.config.yml | 7 ++ .../importer/volume/ig-importer-app.json | 11 +++ .../importer/volume/openhimConfig.js | 79 ++++++++++++++----- .../package-metadata.json | 2 +- kafka-mapper-consumer/consumer-ui-app.json | 11 +++ .../docker-compose.config.yml | 41 ++++++++++ kafka-mapper-consumer/openhimConfig.js | 72 +++++++++++++++++ kafka-mapper-consumer/package-metadata.json | 2 +- kafka-mapper-consumer/swarm.sh | 1 + 10 files changed, 212 insertions(+), 24 deletions(-) create mode 100644 fhir-ig-importer/importer/volume/ig-importer-app.json create mode 100644 kafka-mapper-consumer/consumer-ui-app.json create mode 100644 kafka-mapper-consumer/docker-compose.config.yml create mode 100644 kafka-mapper-consumer/openhimConfig.js diff --git a/config.yaml b/config.yaml index e561e5a9..b54a06c5 100644 --- a/config.yaml +++ b/config.yaml @@ -1,5 +1,5 @@ projectName: platform -image: jembi/platform:latest +image: jembi/platform:3.0.0-beta logPath: /tmp/logs packages: @@ -74,3 +74,11 @@ profiles: - openhim-mapping-mediator envFiles: - mpi.env + + - name: test-recipe + packages: + - fhir-ig-importer + - kafka-mapper-consumer + envFiles: + - cdr-dw.env + dev: true diff --git a/fhir-ig-importer/importer/docker-compose.config.yml b/fhir-ig-importer/importer/docker-compose.config.yml index 2f47ef17..c6398016 100644 --- a/fhir-ig-importer/importer/docker-compose.config.yml +++ b/fhir-ig-importer/importer/docker-compose.config.yml @@ -18,6 +18,8 @@ services: target: /openhimConfig.js - source: fhir-ig-importer-config-importer-openhim-import.json target: /openhim-import.json + - source: fhir-ig-importer-config-importer-ig-importer-app.json + target: /ig-importer-app.json deploy: replicas: 1 restart_policy: @@ -34,6 +36,11 @@ configs: name: fhir-ig-importer-config-importer-openhim-import.json-${fhir_ig_importer_config_importer_openhim_import_js_DIGEST:?err} labels: name: fhir-ig-importer + fhir-ig-importer-config-importer-ig-importer-app.json: + file: ./volume/ig-importer-app.json + name: fhir-ig-importer-config-importer-ig-importer-app.json-${fhir_ig_importer_config_importer_ig_importer_app_DIGEST:?err} + labels: + name: fhir-ig-importer networks: openhim: diff --git a/fhir-ig-importer/importer/volume/ig-importer-app.json b/fhir-ig-importer/importer/volume/ig-importer-app.json new file mode 100644 index 00000000..a86b0cc8 --- /dev/null +++ b/fhir-ig-importer/importer/volume/ig-importer-app.json @@ -0,0 +1,11 @@ +{ + "name": "FHIR IG Importer", + "description": "FHIR IG microfrontend app", + "category": "HIE Configuration", + "type": "esmodule", + "url": "http://localhost:3000/jembi-fhir-ig-importer.js", + "showInPortal": true, + "showInSideBar": true, + "access_roles": ["admin"], + "icon": "https://fonts.gstatic.com/s/i/materialicons/medical_information/v1/24px.svg" +} diff --git a/fhir-ig-importer/importer/volume/openhimConfig.js b/fhir-ig-importer/importer/volume/openhimConfig.js index 182ee887..cae7a270 100644 --- a/fhir-ig-importer/importer/volume/openhimConfig.js +++ b/fhir-ig-importer/importer/volume/openhimConfig.js @@ -1,53 +1,90 @@ -"use strict"; - const fs = require("fs"); const https = require("https"); const path = require("path"); +("use strict"); + const OPENHIM_CORE_SERVICE_NAME = "openhim-core"; const OPENHIM_MEDIATOR_API_PORT = 8080; const OPENHIM_API_PASSWORD = process.env.OPENHIM_API_PASSWORD || "instant101"; const OPENHIM_API_USERNAME = process.env.OPENHIM_API_USERNAME || "root@openhim.org"; -const authHeader = new Buffer.from( +const authHeader = Buffer.from( `${OPENHIM_API_USERNAME}:${OPENHIM_API_PASSWORD}` ).toString("base64"); +function makeRequest(options, data) { + const req = https.request(options, (res) => { + if (res.statusCode == 401) { + throw new Error(`Incorrect OpenHIM API credentials`); + } + + if (![201, 200].includes(res.statusCode)) { + throw new Error(`Failed to import OpenHIM config: ${res.statusCode}`); + } + + console.log("Successfully Imported OpenHIM Config"); + }); + + req.on("error", (error) => { + throw new Error(`Failed to import OpenHIM config: ${error}`); + }); + + req.write(data); + req.end(); +} + const jsonData = JSON.parse( fs.readFileSync(path.resolve(__dirname, "openhim-import.json")) ); +const appJsonData = JSON.parse( + fs.readFileSync(path.resolve(__dirname, "ig-importer-app.json")) +); + const data = JSON.stringify(jsonData); +const appData = JSON.stringify(appJsonData); const options = { protocol: "https:", hostname: OPENHIM_CORE_SERVICE_NAME, port: OPENHIM_MEDIATOR_API_PORT, - path: "/metadata", - method: "POST", headers: { "Content-Type": "application/json", - "Content-Length": data.length, Authorization: `Basic ${authHeader}`, }, }; -const req = https.request(options, (res) => { - if (res.statusCode == 401) { - throw new Error(`Incorrect OpenHIM API credentials`); - } - - if (res.statusCode != 201) { - throw new Error(`Failed to import OpenHIM config: ${res.statusCode}`); - } +const reqOptions = { + ...options, + path: "/metadata", + method: "POST", + headers: { + ...options.headers, + "Content-Length": data.length, + }, +}; - console.log("Successfully Imported OpenHIM Config"); -}); +const appReqOptions = { + ...options, + path: "/apps", + method: "POST", + headers: { + ...options.headers, + "Content-Length": appData.length, + }, +}; -req.on("error", (error) => { - throw new Error(`Failed to import OpenHIM config: ${error}`); -}); +const importMapRebuildOptions = { + ...options, + path: "/apps", + method: "GET", + headers: { + ...options.headers, + }, +}; -req.write(data); -req.end(); +makeRequest(reqOptions, data); +makeRequest(appReqOptions, appData); +makeRequest(importMapRebuildOptions, ""); diff --git a/interoperability-layer-openhim/package-metadata.json b/interoperability-layer-openhim/package-metadata.json index 9bfef8e2..1d8974b1 100644 --- a/interoperability-layer-openhim/package-metadata.json +++ b/interoperability-layer-openhim/package-metadata.json @@ -6,7 +6,7 @@ "type": "infrastructure", "dependencies": [], "environmentVariables": { - "OPENHIM_CORE_IMAGE": "jembi/openhim-core:microfrontends-pre-release", + "OPENHIM_CORE_IMAGE": "jembi/openhim-core:microfrontends-3.0.0-beta", "MONGO_SET_COUNT": "1", "OPENHIM_CORE_INSTANCES": "1", "OPENHIM_CONSOLE_INSTANCES": "1", diff --git a/kafka-mapper-consumer/consumer-ui-app.json b/kafka-mapper-consumer/consumer-ui-app.json new file mode 100644 index 00000000..5d200039 --- /dev/null +++ b/kafka-mapper-consumer/consumer-ui-app.json @@ -0,0 +1,11 @@ +{ + "name": "Kafka Mapper Consumer", + "description": "Kafka mapper consumer microfrontents app", + "category": "HIE Configuration", + "type": "esmodule", + "url": "http://localhost:8091/jembi-kafka-mapper-consumer-ui.js", + "showInPortal": true, + "showInSideBar": false, + "access_roles": ["admin"], + "icon": "https://fonts.gstatic.com/s/i/materialicons/apps/v12/24px.svg" +} diff --git a/kafka-mapper-consumer/docker-compose.config.yml b/kafka-mapper-consumer/docker-compose.config.yml new file mode 100644 index 00000000..168dd4b8 --- /dev/null +++ b/kafka-mapper-consumer/docker-compose.config.yml @@ -0,0 +1,41 @@ +version: '3.9' + +services: + # container for executing config import scripts for creating the OpenHIM channels used by the Mediator + kafka-mapper-consumer-config-importer: + image: node:erbium-alpine + networks: + openhim: + default: + environment: + OPENHIM_API_USERNAME: ${OPENHIM_USERNAME} + OPENHIM_API_PASSWORD: ${OPENHIM_PASSWORD} + # Reject unauthorised is only needed if the OpenHIM's SSL is not setup + NODE_TLS_REJECT_UNAUTHORIZED: 0 + command: sh -c "node openhimConfig.js" + configs: + - source: kafka-mapper-consumer-openhimConfig.js + target: /openhimConfig.js + - source: kafka-mapper-consumer-consumer-ui-app.json + target: /consumer-ui-app.json + deploy: + replicas: 1 + restart_policy: + condition: none + +configs: + kafka-mapper-consumer-openhimConfig.js: + file: ./openhimConfig.js + name: kafka-mapper-consumer-openhimConfig.js-${fhir_ig_importer_config_importer_openhimConfig_js_DIGEST:?err} + labels: + name: kafka-mapper-consumer + kafka-mapper-consumer-consumer-ui-app.json: + file: ./consumer-ui-app.json + name: kafka-mapper-consumer-consumer-ui-app.json-${kafka_mapper_consumer_ui_json_DIGEST:?err} + labels: + name: kafka-mapper-consumer + +networks: + openhim: + name: openhim_public + external: true diff --git a/kafka-mapper-consumer/openhimConfig.js b/kafka-mapper-consumer/openhimConfig.js new file mode 100644 index 00000000..a7868cde --- /dev/null +++ b/kafka-mapper-consumer/openhimConfig.js @@ -0,0 +1,72 @@ +const fs = require("fs"); +const https = require("https"); +const path = require("path"); + +("use strict"); + +const OPENHIM_CORE_SERVICE_NAME = "openhim-core"; +const OPENHIM_MEDIATOR_API_PORT = 8080; +const OPENHIM_API_PASSWORD = process.env.OPENHIM_API_PASSWORD || "instant101"; +const OPENHIM_API_USERNAME = + process.env.OPENHIM_API_USERNAME || "root@openhim.org"; + +const authHeader = Buffer.from( + `${OPENHIM_API_USERNAME}:${OPENHIM_API_PASSWORD}` +).toString("base64"); +function makeRequest(options, data) { + const req = https.request(options, (res) => { + if (res.statusCode == 401) { + throw new Error(`Incorrect OpenHIM API credentials`); + } + + if (![201, 200].includes(res.statusCode)) { + throw new Error(`Failed to import OpenHIM config: ${res.statusCode}`); + } + + console.log("Successfully Imported OpenHIM Config"); + }); + + req.on("error", (error) => { + throw new Error(`Failed to import OpenHIM config: ${error}`); + }); + + req.write(data); + req.end(); +} + +const appJsonData = JSON.parse( + fs.readFileSync(path.resolve(__dirname, "consumer-ui-app.json")) +); +const appData = JSON.stringify(appJsonData); + +const options = { + protocol: "https:", + hostname: OPENHIM_CORE_SERVICE_NAME, + port: OPENHIM_MEDIATOR_API_PORT, + headers: { + "Content-Type": "application/json", + Authorization: `Basic ${authHeader}`, + }, +}; + +const appReqOptions = { + ...options, + path: "/apps", + method: "POST", + headers: { + ...options.headers, + "Content-Length": appData.length, + }, +}; + +const importMapRebuildOptions = { + ...options, + path: "/apps", + method: "GET", + headers: { + ...options.headers, + }, +}; + +makeRequest(appReqOptions, appData); +makeRequest(importMapRebuildOptions, ""); diff --git a/kafka-mapper-consumer/package-metadata.json b/kafka-mapper-consumer/package-metadata.json index 57fc4199..a28cd5da 100644 --- a/kafka-mapper-consumer/package-metadata.json +++ b/kafka-mapper-consumer/package-metadata.json @@ -18,6 +18,6 @@ "CLICKHOUSE_HOST": "analytics-datastore-clickhouse", "CLICKHOUSE_PORT": "8123", "KAFKA_CONSUMER_MAPPER_MEDIATOR_VERSION": "jembi/kafka-mapper-consumer:0.1.0", - "KAFKA_CONSUMER_MAPPER_UI_VERSION": "jembi/kafka-mapper-consumer-ui:0.1.0-alpha" + "KAFKA_CONSUMER_MAPPER_UI_VERSION": "jembi/kafka-mapper-consumer-ui:0.1.1-alpha" } } diff --git a/kafka-mapper-consumer/swarm.sh b/kafka-mapper-consumer/swarm.sh index 8d1f9ed6..dbf5e83d 100755 --- a/kafka-mapper-consumer/swarm.sh +++ b/kafka-mapper-consumer/swarm.sh @@ -45,6 +45,7 @@ function initialize_package() { log error "Failed to deploy package" exit 1 } + docker::deploy_config_importer $STACK "$COMPOSE_FILE_PATH/docker-compose.config.yml" "kafka-mapper-consumer-config-importer" "kafka-mapper-consumer" } function destroy_package() { From 918c7f97c95212079ccd0bfa2c88cdac1114d16b Mon Sep 17 00:00:00 2001 From: bradsawadye Date: Thu, 9 May 2024 15:03:54 +0200 Subject: [PATCH 13/43] fix breaking test --- test/cucumber/features/single-mode/recipe.feature | 1 - 1 file changed, 1 deletion(-) diff --git a/test/cucumber/features/single-mode/recipe.feature b/test/cucumber/features/single-mode/recipe.feature index 54a93c50..2de872d0 100644 --- a/test/cucumber/features/single-mode/recipe.feature +++ b/test/cucumber/features/single-mode/recipe.feature @@ -12,7 +12,6 @@ Scenario: Init the CDR recipe And The service "kafka-01" should be started with 1 replica And The service "kafdrop" should be started with 1 replica And The service "kafka-minion" should be started with 1 replica - And The service "keycloak-postgres-1" should be started with 1 replica And The service "identity-access-manager-keycloak" should be started with 1 replica And The service "jempi-ratel" should be started with 1 replica And The service "jempi-alpha-01" should be started with 1 replica From 909cc9e11df7c839598625889baf41a85f8a4a69 Mon Sep 17 00:00:00 2001 From: bradsawadye Date: Thu, 9 May 2024 15:06:03 +0200 Subject: [PATCH 14/43] Clean Remove unused files --- .../docker-compose-postgres.cluster.yml | 74 ------------------- .../docker-compose-postgres.dev.yml | 8 -- .../docker-compose-postgres.yml | 40 ---------- 3 files changed, 122 deletions(-) delete mode 100644 identity-access-manager-keycloak/docker-compose-postgres.cluster.yml delete mode 100644 identity-access-manager-keycloak/docker-compose-postgres.dev.yml delete mode 100644 identity-access-manager-keycloak/docker-compose-postgres.yml diff --git a/identity-access-manager-keycloak/docker-compose-postgres.cluster.yml b/identity-access-manager-keycloak/docker-compose-postgres.cluster.yml deleted file mode 100644 index 4be183d9..00000000 --- a/identity-access-manager-keycloak/docker-compose-postgres.cluster.yml +++ /dev/null @@ -1,74 +0,0 @@ -version: "3.9" - -services: - keycloak-postgres-1: - environment: - REPMGR_PARTNER_NODES: ${KC_REPMGR_PARTNER_NODES} - deploy: - placement: - constraints: - - "node.labels.name==${KEYCLOAK_POSTGRES_1}" - - keycloak-postgres-2: - image: ${POSTGRES_IMAGE} - environment: - POSTGRESQL_PASSWORD: ${KC_POSTGRESQL_PASSWORD} - POSTGRESQL_USERNAME: ${KC_POSTGRESQL_USERNAME} - POSTGRESQL_DATABASE: ${KC_POSTGRESQL_DATABASE} - REPMGR_NODE_NETWORK_NAME: keycloak-postgres-2 - REPMGR_PASSWORD: ${KC_REPMGR_PASSWORD} - REPMGR_RECONNECT_INTERVAL: 3 - REPMGR_NODE_NAME: keycloak-postgres-2 - REPMGR_PRIMARY_HOST: ${KC_REPMGR_PRIMARY_HOST} - REPMGR_PARTNER_NODES: ${KC_REPMGR_PARTNER_NODES} - volumes: - - "keycloak-postgres-2-data:/bitnami/postgresql" - deploy: - placement: - constraints: - - "node.labels.name==${KEYCLOAK_POSTGRES_2}" - replicas: ${POSTGRES_REPLICAS} - resources: - limits: - cpus: ${KC_POSTGRES_CPU_LIMIT} - memory: ${KC_POSTGRES_MEMORY_LIMIT} - reservations: - cpus: ${KC_POSTGRES_CPU_RESERVE} - memory: ${KC_POSTGRES_MEMORY_RESERVE} - networks: - default: - keycloak_backup_net: {} - - keycloak-postgres-3: - image: ${POSTGRES_IMAGE} - environment: - POSTGRESQL_PASSWORD: ${KC_POSTGRESQL_PASSWORD} - POSTGRESQL_USERNAME: ${KC_POSTGRESQL_USERNAME} - POSTGRESQL_DATABASE: ${KC_POSTGRESQL_DATABASE} - REPMGR_NODE_NETWORK_NAME: keycloak-postgres-3 - REPMGR_PASSWORD: ${KC_REPMGR_PASSWORD} - REPMGR_RECONNECT_INTERVAL: 3 - REPMGR_NODE_NAME: keycloak-postgres-3 - REPMGR_PRIMARY_HOST: ${KC_REPMGR_PRIMARY_HOST} - REPMGR_PARTNER_NODES: ${KC_REPMGR_PARTNER_NODES} - volumes: - - "keycloak-postgres-3-data:/bitnami/postgresql" - deploy: - placement: - constraints: - - "node.labels.name==${KEYCLOAK_POSTGRES_3}" - replicas: ${POSTGRES_REPLICAS} - resources: - limits: - cpus: ${KC_POSTGRES_CPU_LIMIT} - memory: ${KC_POSTGRES_MEMORY_LIMIT} - reservations: - cpus: ${KC_POSTGRES_CPU_RESERVE} - memory: ${KC_POSTGRES_MEMORY_RESERVE} - networks: - default: - keycloak_backup_net: {} - -volumes: - keycloak-postgres-2-data: - keycloak-postgres-3-data: diff --git a/identity-access-manager-keycloak/docker-compose-postgres.dev.yml b/identity-access-manager-keycloak/docker-compose-postgres.dev.yml deleted file mode 100644 index 79e221ae..00000000 --- a/identity-access-manager-keycloak/docker-compose-postgres.dev.yml +++ /dev/null @@ -1,8 +0,0 @@ -version: '3.9' - -services: - keycloak-postgres-1: - ports: - - target: 5432 - published: 5434 - mode: host diff --git a/identity-access-manager-keycloak/docker-compose-postgres.yml b/identity-access-manager-keycloak/docker-compose-postgres.yml deleted file mode 100644 index 4fa41d41..00000000 --- a/identity-access-manager-keycloak/docker-compose-postgres.yml +++ /dev/null @@ -1,40 +0,0 @@ -version: "3.9" - -services: - keycloak-postgres-1: - image: ${POSTGRES_IMAGE} - environment: - POSTGRESQL_PASSWORD: ${KC_POSTGRESQL_PASSWORD} - POSTGRESQL_USERNAME: ${KC_POSTGRESQL_USERNAME} - POSTGRESQL_DATABASE: ${KC_POSTGRESQL_DATABASE} - POSTGRESQL_POSTGRES_PASSWORD: ${KC_POSTGRESQL_PASSWORD} - REPMGR_NODE_NETWORK_NAME: keycloak-postgres-1 - REPMGR_PASSWORD: ${KC_REPMGR_PASSWORD} - REPMGR_RECONNECT_INTERVAL: 3 - REPMGR_NODE_NAME: keycloak-postgres-1 - REPMGR_PRIMARY_HOST: ${KC_REPMGR_PRIMARY_HOST} - REPMGR_PARTNER_NODES: ${KC_REPMGR_PARTNER_NODES} - volumes: - - "keycloak-postgres-1-data:/bitnami/postgresql" - deploy: - replicas: ${POSTGRES_REPLICAS} - resources: - limits: - cpus: ${KC_POSTGRES_CPU_LIMIT} - memory: ${KC_POSTGRES_MEMORY_LIMIT} - reservations: - cpus: ${KC_POSTGRES_CPU_RESERVE} - memory: ${KC_POSTGRES_MEMORY_RESERVE} - networks: - default: - keycloak_backup_net: {} - -volumes: - keycloak-postgres-1-data: - -networks: - default: - keycloak_backup_net: - name: keycloak_backup - driver: overlay - attachable: true From 800a8e294cd7f534395ac1bddcf1e561f5423cdb Mon Sep 17 00:00:00 2001 From: Mohamed Marrouchi Date: Thu, 9 May 2024 14:26:48 +0100 Subject: [PATCH 15/43] Update README.md --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index bf2dba53..1a2272ca 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,13 @@ Each service's resource allocations can be piped into their .yml file through en - Take note to not allocate less memory to ELK Stack services than their JVM heap sizes. - Exit code 137 indicates an out-of-memory failure. When running into this, it means that the service has been allocated too little memory. +## Build multi-platform docker images +It's essential to make sure that any docker image should be available for multiple platforms : AMD, ARM, ... (not only linux, but MacOS as well). To do so you can follow the steps below : +1. Create your own custom builder by running `docker buildx create --name mycustombuilder --driver docker-container --bootstrap` +2. Ask docker to use this new builder for future builds by running `docker buildx use mycustombuilder` +3. Inspect buildx to see if docker has indeed switched builders to the new one you asked it to use by running `docker buildx inspect` +4. Then you can perform the build and push, for example : `docker buildx build --platform linux/amd64,linux/arm64 --push -t jembi/hapi:v7.0.3-wget .` + ## Tests Tests are located in `/test` From 154805c671a6c694698487bbe23274701c35ef12 Mon Sep 17 00:00:00 2001 From: drono Date: Tue, 14 May 2024 10:19:35 +0300 Subject: [PATCH 16/43] Fix single node e2e tests --- test/cucumber/features/single-mode/recipe.feature | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/test/cucumber/features/single-mode/recipe.feature b/test/cucumber/features/single-mode/recipe.feature index 54a93c50..c676be79 100644 --- a/test/cucumber/features/single-mode/recipe.feature +++ b/test/cucumber/features/single-mode/recipe.feature @@ -1,7 +1,7 @@ Feature: CDR-DW recipe? Does the recipe work as expected -Scenario: Init the CDR recipe + Scenario: Init the CDR recipe Given I use parameters "package init -p cdr-dw --dev --env-file=cdr-dw.env" When I launch the platform with params Then The service "mongo-1" should be started with 1 replica @@ -12,7 +12,6 @@ Scenario: Init the CDR recipe And The service "kafka-01" should be started with 1 replica And The service "kafdrop" should be started with 1 replica And The service "kafka-minion" should be started with 1 replica - And The service "keycloak-postgres-1" should be started with 1 replica And The service "identity-access-manager-keycloak" should be started with 1 replica And The service "jempi-ratel" should be started with 1 replica And The service "jempi-alpha-01" should be started with 1 replica @@ -35,27 +34,27 @@ Scenario: Init the CDR recipe And The service "dashboard-visualiser-superset" should be started with 1 replica And The service "analytics-datastore-clickhouse" should be started with 1 replica -Scenario: Send Fhir bundle and store the clinical data in the Fhir datastore, the patient info in the CR + Scenario: Send Fhir bundle and store the clinical data in the Fhir datastore, the patient info in the CR Given I have configured the cdr When I send a fhir patient bundle Then the clinical data should be stored in hapi fhir And the patient data in the Jempi client registry And the data should be stored in clickhouse -Scenario: Fetch International Patient summary (IPS) + Scenario: Fetch International Patient summary (IPS) When I then send a fhir patient summary request Then I should get a successful summary response -Scenario: Fetch everything for a patient (all the clinical data) + Scenario: Fetch everything for a patient (all the clinical data) When I then send a request for all the patient's clinical data Then I should get a successful everything response -Scenario: Bring down the servers + Scenario: Bring down the servers Given I use parameters "package down -p cdr-dw --env-file=cdr-dw.env" When I launch the platform with params Then a request to fetch data from the cdr should fail -Scenario: Bring up the servers and test + Scenario: Bring up the servers and test Given I use parameters "package up -p cdr-dw --dev --env-file=cdr-dw.env" When I launch the platform with params Then The service "mongo-1" should be started with 1 replica @@ -67,7 +66,7 @@ Scenario: Bring up the servers and test Then I should get a successful everything response And the data should be stored in clickhouse -Scenario: Destroy the services + Scenario: Destroy the services Given I use parameters "package remove -p cdr-dw --env-file=cdr-dw.env" When I launch the platform with params Then There should be 0 service From 57cee7da09713313cfb165a036551d5b7f434685 Mon Sep 17 00:00:00 2001 From: Drizzentic Date: Tue, 14 May 2024 12:39:07 +0300 Subject: [PATCH 17/43] Update kafka-mapper-consumer/consumer-ui-app.json Co-authored-by: Martin Brocker --- kafka-mapper-consumer/consumer-ui-app.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kafka-mapper-consumer/consumer-ui-app.json b/kafka-mapper-consumer/consumer-ui-app.json index 5d200039..b6ac02d7 100644 --- a/kafka-mapper-consumer/consumer-ui-app.json +++ b/kafka-mapper-consumer/consumer-ui-app.json @@ -1,6 +1,6 @@ { "name": "Kafka Mapper Consumer", - "description": "Kafka mapper consumer microfrontents app", + "description": "Kafka mapper consumer microfrontends app", "category": "HIE Configuration", "type": "esmodule", "url": "http://localhost:8091/jembi-kafka-mapper-consumer-ui.js", From 226945cee0de529f7df1bf8be5b8de510ba6dbb9 Mon Sep 17 00:00:00 2001 From: ItsMurumba Date: Tue, 14 May 2024 16:59:49 +0300 Subject: [PATCH 18/43] Cleanup --- config.yaml | 12 ++++++++++++ interoperability-layer-openhim/docker-compose.yml | 2 +- interoperability-layer-openhim/package-metadata.json | 2 +- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/config.yaml b/config.yaml index 784faeac..4681b2b6 100644 --- a/config.yaml +++ b/config.yaml @@ -75,3 +75,15 @@ profiles: envFiles: - mpi.env + - name: dev + packages: + - interoperability-layer-openhim + - reprocess-mediator + # - reverse-proxy-nginx + # - fhir-datastore-hapi-fhir + # - monitoring + # - identity-access-manager-keycloak + envFiles: + - .env + dev: true + diff --git a/interoperability-layer-openhim/docker-compose.yml b/interoperability-layer-openhim/docker-compose.yml index c6eed4b2..abb32bb8 100644 --- a/interoperability-layer-openhim/docker-compose.yml +++ b/interoperability-layer-openhim/docker-compose.yml @@ -2,7 +2,7 @@ version: "3.9" services: openhim-core: - image: jembi/openhim-core:v8.4.0 + image: jembi/openhim-core:latest networks: kafka: hapi-fhir: diff --git a/interoperability-layer-openhim/package-metadata.json b/interoperability-layer-openhim/package-metadata.json index ac643e01..885a06fe 100644 --- a/interoperability-layer-openhim/package-metadata.json +++ b/interoperability-layer-openhim/package-metadata.json @@ -26,7 +26,7 @@ "OPENHIM_MONGO_MEMORY_RESERVE": "500M", "OPENHIM_MONGO_URL": "mongodb://mongo-1:27017/openhim", "OPENHIM_MONGO_ATNAURL": "mongodb://mongo-1:27017/openhim", - "OPENHIM_CONSOLE_VERSION": "jembi/openhim-console:v1.18.2", + "OPENHIM_CONSOLE_VERSION": "jembi/openhim-console:microfrontend-poc", "KAFKA_HOSTS": "kafka-01:9092", "KC_REALM_NAME": "platform-realm", "KC_FRONTEND_URL": "http://localhost:9088", From e0d486033473c9f85647ac65686517bda40d8921 Mon Sep 17 00:00:00 2001 From: Drizzentic Date: Tue, 14 May 2024 17:33:16 +0300 Subject: [PATCH 19/43] Update documentation/recipes/README.md Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- documentation/recipes/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/recipes/README.md b/documentation/recipes/README.md index 9e792ad2..842d6331 100644 --- a/documentation/recipes/README.md +++ b/documentation/recipes/README.md @@ -5,6 +5,7 @@ description: Pre-defined recipes for common use cases # 📜 Recipes OpenHIM platform comes bundles with a set of generic packages that can be deployed and configured to support a number of different use cases. To help users of OpenHIM Platform get started with something they can make use of immediately, a number of default OpenHIM Platform recipes are provided. These help you get started with everything you need setup and configured for a particular use case. +We currently support the following default recipes: We current support the following default recipes: From 18093c0e1ed258a6249eb615898625998b48fd54 Mon Sep 17 00:00:00 2001 From: Drizzentic Date: Wed, 15 May 2024 10:12:51 +0300 Subject: [PATCH 20/43] Update documentation/recipes/README.md Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- documentation/recipes/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/recipes/README.md b/documentation/recipes/README.md index 842d6331..c31add42 100644 --- a/documentation/recipes/README.md +++ b/documentation/recipes/README.md @@ -4,7 +4,7 @@ description: Pre-defined recipes for common use cases # 📜 Recipes -OpenHIM platform comes bundles with a set of generic packages that can be deployed and configured to support a number of different use cases. To help users of OpenHIM Platform get started with something they can make use of immediately, a number of default OpenHIM Platform recipes are provided. These help you get started with everything you need setup and configured for a particular use case. +OpenHIM platform comes bundled with a set of generic packages that can be deployed and configured to support a number of different use cases. To help users of OpenHIM Platform get started with something they can make use of immediately, a number of default OpenHIM Platform recipes are provided. These help you get started with everything you need setup and configured for a particular use case. We currently support the following default recipes: We current support the following default recipes: From 05da7de012cec0089d6fd61fe8e1e40e8e0eccdf Mon Sep 17 00:00:00 2001 From: Drizzentic Date: Wed, 15 May 2024 10:13:00 +0300 Subject: [PATCH 21/43] Update documentation/recipes/README.md Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- documentation/recipes/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/recipes/README.md b/documentation/recipes/README.md index c31add42..8076f5f4 100644 --- a/documentation/recipes/README.md +++ b/documentation/recipes/README.md @@ -3,7 +3,7 @@ description: Pre-defined recipes for common use cases --- # 📜 Recipes - +description: Pre-defined recipes for the common use cases OpenHIM platform comes bundled with a set of generic packages that can be deployed and configured to support a number of different use cases. To help users of OpenHIM Platform get started with something they can make use of immediately, a number of default OpenHIM Platform recipes are provided. These help you get started with everything you need setup and configured for a particular use case. We currently support the following default recipes: From b35f6e2dd148f106245eed6e2a545186d4903b6c Mon Sep 17 00:00:00 2001 From: Drizzentic Date: Wed, 15 May 2024 11:25:16 +0300 Subject: [PATCH 22/43] Update documentation/recipes/README.md Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- documentation/recipes/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/documentation/recipes/README.md b/documentation/recipes/README.md index 8076f5f4..e2e83d19 100644 --- a/documentation/recipes/README.md +++ b/documentation/recipes/README.md @@ -2,7 +2,12 @@ description: Pre-defined recipes for common use cases --- +--- +description: Pre-defined recipes for the common use cases +--- + # 📜 Recipes + description: Pre-defined recipes for the common use cases OpenHIM platform comes bundled with a set of generic packages that can be deployed and configured to support a number of different use cases. To help users of OpenHIM Platform get started with something they can make use of immediately, a number of default OpenHIM Platform recipes are provided. These help you get started with everything you need setup and configured for a particular use case. We currently support the following default recipes: From c25e277aeda80798eac7d507105aa942a215d667 Mon Sep 17 00:00:00 2001 From: Drizzentic Date: Wed, 15 May 2024 11:26:21 +0300 Subject: [PATCH 23/43] Update documentation/recipes/README.md Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- documentation/recipes/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/documentation/recipes/README.md b/documentation/recipes/README.md index e2e83d19..a51feff7 100644 --- a/documentation/recipes/README.md +++ b/documentation/recipes/README.md @@ -12,7 +12,6 @@ description: Pre-defined recipes for the common use cases OpenHIM platform comes bundled with a set of generic packages that can be deployed and configured to support a number of different use cases. To help users of OpenHIM Platform get started with something they can make use of immediately, a number of default OpenHIM Platform recipes are provided. These help you get started with everything you need setup and configured for a particular use case. We currently support the following default recipes: -We current support the following default recipes: From 70b8e91d9199e07bac71d56ae9d078806e01803e Mon Sep 17 00:00:00 2001 From: Drizzentic Date: Wed, 15 May 2024 11:26:36 +0300 Subject: [PATCH 24/43] Update documentation/recipes/README.md Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- documentation/recipes/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/recipes/README.md b/documentation/recipes/README.md index a51feff7..8f3e5010 100644 --- a/documentation/recipes/README.md +++ b/documentation/recipes/README.md @@ -7,7 +7,7 @@ description: Pre-defined recipes for the common use cases --- # 📜 Recipes - +description: Pre-defined recipes for the common use cases description: Pre-defined recipes for the common use cases OpenHIM platform comes bundled with a set of generic packages that can be deployed and configured to support a number of different use cases. To help users of OpenHIM Platform get started with something they can make use of immediately, a number of default OpenHIM Platform recipes are provided. These help you get started with everything you need setup and configured for a particular use case. We currently support the following default recipes: From 6503865fec5dd57abf54a91e9d2f5b1b9250d7dc Mon Sep 17 00:00:00 2001 From: Drizzentic Date: Wed, 15 May 2024 11:26:50 +0300 Subject: [PATCH 25/43] Update documentation/recipes/README.md Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- documentation/recipes/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/recipes/README.md b/documentation/recipes/README.md index 8f3e5010..087373d7 100644 --- a/documentation/recipes/README.md +++ b/documentation/recipes/README.md @@ -2,6 +2,7 @@ description: Pre-defined recipes for common use cases --- +--- --- description: Pre-defined recipes for the common use cases --- From d9b86fba8f90f839d495157ee63af2acb8b1404e Mon Sep 17 00:00:00 2001 From: Drizzentic Date: Wed, 15 May 2024 11:27:10 +0300 Subject: [PATCH 26/43] Update documentation/recipes/README.md Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- documentation/recipes/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/documentation/recipes/README.md b/documentation/recipes/README.md index 087373d7..97b2ceec 100644 --- a/documentation/recipes/README.md +++ b/documentation/recipes/README.md @@ -9,7 +9,9 @@ description: Pre-defined recipes for the common use cases # 📜 Recipes description: Pre-defined recipes for the common use cases + description: Pre-defined recipes for the common use cases + OpenHIM platform comes bundled with a set of generic packages that can be deployed and configured to support a number of different use cases. To help users of OpenHIM Platform get started with something they can make use of immediately, a number of default OpenHIM Platform recipes are provided. These help you get started with everything you need setup and configured for a particular use case. We currently support the following default recipes: From 38ab1766a50689f05b11019c76dc1958dc3af9c9 Mon Sep 17 00:00:00 2001 From: Drizzentic Date: Wed, 15 May 2024 11:27:45 +0300 Subject: [PATCH 27/43] Update documentation/recipes/README.md Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- documentation/recipes/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/documentation/recipes/README.md b/documentation/recipes/README.md index 97b2ceec..12b5de80 100644 --- a/documentation/recipes/README.md +++ b/documentation/recipes/README.md @@ -13,7 +13,6 @@ description: Pre-defined recipes for the common use cases description: Pre-defined recipes for the common use cases OpenHIM platform comes bundled with a set of generic packages that can be deployed and configured to support a number of different use cases. To help users of OpenHIM Platform get started with something they can make use of immediately, a number of default OpenHIM Platform recipes are provided. These help you get started with everything you need setup and configured for a particular use case. -We currently support the following default recipes: From 03941211d3432f38fdfecd535138ce01455b619b Mon Sep 17 00:00:00 2001 From: ItsMurumba Date: Wed, 15 May 2024 13:31:29 +0300 Subject: [PATCH 28/43] Fix conflict on recipe documentation --- documentation/recipes/README.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/documentation/recipes/README.md b/documentation/recipes/README.md index 12b5de80..75b6a8ae 100644 --- a/documentation/recipes/README.md +++ b/documentation/recipes/README.md @@ -2,15 +2,7 @@ description: Pre-defined recipes for common use cases --- ---- ---- -description: Pre-defined recipes for the common use cases ---- - # 📜 Recipes -description: Pre-defined recipes for the common use cases - -description: Pre-defined recipes for the common use cases OpenHIM platform comes bundled with a set of generic packages that can be deployed and configured to support a number of different use cases. To help users of OpenHIM Platform get started with something they can make use of immediately, a number of default OpenHIM Platform recipes are provided. These help you get started with everything you need setup and configured for a particular use case. From 643ed44fe21cee3f572b709f82f8ec6eab0dd35b Mon Sep 17 00:00:00 2001 From: ItsMurumba Date: Wed, 15 May 2024 13:35:49 +0300 Subject: [PATCH 29/43] Fix conflict on recipe documentation --- documentation/recipes/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/recipes/README.md b/documentation/recipes/README.md index 75b6a8ae..199ae251 100644 --- a/documentation/recipes/README.md +++ b/documentation/recipes/README.md @@ -1,3 +1,4 @@ + --- description: Pre-defined recipes for common use cases --- From ff712698342859411f832d3ab5521f54219ffa45 Mon Sep 17 00:00:00 2001 From: ItsMurumba Date: Wed, 15 May 2024 13:38:49 +0300 Subject: [PATCH 30/43] Fix conflict on recipe documentation --- documentation/recipes/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/documentation/recipes/README.md b/documentation/recipes/README.md index 199ae251..b10d3a91 100644 --- a/documentation/recipes/README.md +++ b/documentation/recipes/README.md @@ -7,7 +7,6 @@ description: Pre-defined recipes for common use cases OpenHIM platform comes bundled with a set of generic packages that can be deployed and configured to support a number of different use cases. To help users of OpenHIM Platform get started with something they can make use of immediately, a number of default OpenHIM Platform recipes are provided. These help you get started with everything you need setup and configured for a particular use case. - - +We currently support the following default recipes:
Central Data Repository with Data WarehouseA FHIR-based Shared Health record linked to an MPI for linking and matching patient demographics and a default reporting pipeline to transform and visualise FHIR data.central-data-repository-with-data-warehousing.md
Central Data RepositoryA FHIR-based Shared Health record linked to an MPI for linking and matching patient demographics. No reporting is include but all FHIR data is pushed to Kafka for external system to use.central-data-repository-no-reporting.md
Master Patient IndexA master patient index setup using JeMPI. it also includes OpenHIM as the API gateway providing security, a mapping mediator to allow FHIR-based communication with JeMPI and Keycloak to support user management.master-patient-index.md
From 3f8693a37232e3d495adac448ac52b117110492b Mon Sep 17 00:00:00 2001 From: ItsMurumba Date: Wed, 15 May 2024 14:22:10 +0300 Subject: [PATCH 31/43] Address PR Comments and code cleanup --- config.yaml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/config.yaml b/config.yaml index 4dd8d7eb..54977b16 100644 --- a/config.yaml +++ b/config.yaml @@ -76,18 +76,3 @@ profiles: envFiles: - mpi.env - - name: dev - packages: - - reprocess-mediator - envFiles: - - .env.local - dev: true - only: false - - - name: test-recipe - packages: - - fhir-ig-importer - - kafka-mapper-consumer - envFiles: - - cdr-dw.env - dev: true From 3319148c90a16480a6b9bdf6a255d4e0b79a231e Mon Sep 17 00:00:00 2001 From: bradsawadye Date: Wed, 15 May 2024 13:32:21 +0200 Subject: [PATCH 32/43] Temporarily disable clustered mode for jempi as it is not working --- .env.cluster | 6 +++--- client-registry-jempi/swarm.sh | 11 ++++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.env.cluster b/.env.cluster index b0f144a9..a1215e27 100644 --- a/.env.cluster +++ b/.env.cluster @@ -88,18 +88,18 @@ GF_SERVER_DOMAIN=grafana.domain KC_POSTGRES_REPLICA_SET=pgpool-1:5432,pgpool-2:5432,pgpool-3:5432 # Client Registry - JeMPI -JEMPI_WEB_INSTANCES=3 +JEMPI_WEB_INSTANCES=1 REACT_APP_JEMPI_BASE_API_HOST=https://jempi-api.domain REACT_APP_JEMPI_BASE_API_PORT=50000 JEMPI_SESSION_SECURE=true -JEMPI_REPMGR_PARTNER_NODES=jempi-postgresql-01,jempi-postgresql-02,jempi-postgresql-03 +JEMPI_REPMGR_PARTNER_NODES=jempi-postgresql-01 JEMPI_ASYNC_RECEIVER_INSTANCES=1 JEMPI_SYNC_RECEIVER_INSTANCES=1 JEMPI_PRE_PROCESSOR_INSTANCES=1 JEMPI_CONTROLLER_INSTANCES=1 JEMPI_EM_CALCULATOR_INSTANCES=1 JEMPI_LINKER_INSTANCES=1 -JEMPI_API_INSTANCES=3 +JEMPI_API_INSTANCES=1 # Resource limits OPENHIM_MEMORY_LIMIT=4G diff --git a/client-registry-jempi/swarm.sh b/client-registry-jempi/swarm.sh index d1323691..92e57aa2 100644 --- a/client-registry-jempi/swarm.sh +++ b/client-registry-jempi/swarm.sh @@ -52,11 +52,12 @@ function initialize_package() { log info "Running package in PROD mode" fi - if [[ "$CLUSTERED_MODE" == "true" ]]; then - dgraph_cluster_compose_param="docker-compose.dgraph-cluster.yml" - dgraph_zero_cluster_compose_param="docker-compose.dgraph-zero-cluster.yml" - combined_cluster_compose_param="docker-compose.combined-cluster.yml" - fi + # Jempi not working in clustered mode, temporarily disable + # if [[ "$CLUSTERED_MODE" == "true" ]]; then + # dgraph_cluster_compose_param="docker-compose.dgraph-cluster.yml" + # dgraph_zero_cluster_compose_param="docker-compose.dgraph-zero-cluster.yml" + # combined_cluster_compose_param="docker-compose.combined-cluster.yml" + # fi ( log info "Importing JeMPI Kafka topics" From 21eacab7645294273e76a14806049ca3275f801c Mon Sep 17 00:00:00 2001 From: bradsawadye Date: Wed, 15 May 2024 13:52:17 +0200 Subject: [PATCH 33/43] Jempi only runs in single mode ata the moment --- .../features/cluster-mode/jempi.cluster.feature | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/test/cucumber/features/cluster-mode/jempi.cluster.feature b/test/cucumber/features/cluster-mode/jempi.cluster.feature index 81279549..96990800 100644 --- a/test/cucumber/features/cluster-mode/jempi.cluster.feature +++ b/test/cucumber/features/cluster-mode/jempi.cluster.feature @@ -23,8 +23,6 @@ Feature: Client Registry JeMPI? And The service "jempi-alpha-02" should be started with 1 replica And The service "jempi-alpha-03" should be started with 1 replica And The service "jempi-zero-01" should be started with 1 replica - And The service "jempi-zero-02" should be started with 1 replica - And The service "jempi-zero-03" should be started with 1 replica And The service "jempi-async-receiver" should be started with 1 replica And The service "jempi-async-receiver" should be connected to the networks | kafka_public | jempi_default | @@ -41,12 +39,10 @@ Feature: Client Registry JeMPI? And The service "jempi-linker" should be started with 1 replica And The service "jempi-linker" should be connected to the networks | kafka_public | jempi_default | - And The service "jempi-api" should be started with 3 replica + And The service "jempi-api" should be started with 1 replica And The service "jempi-api" should be connected to the networks | kafka_public | jempi_default | And The service "jempi-postgresql-01" should be started with 1 replica - And The service "jempi-postgresql-02" should be started with 1 replica - And The service "jempi-postgresql-03" should be started with 1 replica And The service "jempi-web" should be started with 3 replica And The service "jempi-web" should be connected to the networks | reverse-proxy_public | keycloak_public | jempi_default | @@ -74,13 +70,9 @@ Feature: Client Registry JeMPI? And The service "jempi-em-calculator" should be removed And The service "jempi-linker" should be removed And The service "jempi-zero-01" should be removed - And The service "jempi-zero-02" should be removed - And The service "jempi-zero-03" should be removed And The service "jempi-api" should be removed And The service "jempi-web" should be removed And The service "jempi-postgresql-01" should be removed - And The service "jempi-postgresql-02" should be removed - And The service "jempi-postgresql-03" should be removed And The service "mongo-1" should be removed And The service "mongo-2" should be removed And The service "mongo-3" should be removed From 98d6e135365d93c4e88c4afbf360f1470d0ee93d Mon Sep 17 00:00:00 2001 From: bradsawadye Date: Wed, 15 May 2024 16:14:53 +0200 Subject: [PATCH 34/43] Remove repeating of test Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- test/cucumber/features/cluster-mode/jempi.cluster.feature | 1 + 1 file changed, 1 insertion(+) diff --git a/test/cucumber/features/cluster-mode/jempi.cluster.feature b/test/cucumber/features/cluster-mode/jempi.cluster.feature index 96990800..83210fd8 100644 --- a/test/cucumber/features/cluster-mode/jempi.cluster.feature +++ b/test/cucumber/features/cluster-mode/jempi.cluster.feature @@ -40,6 +40,7 @@ Feature: Client Registry JeMPI? And The service "jempi-linker" should be connected to the networks | kafka_public | jempi_default | And The service "jempi-api" should be started with 1 replica + And The service "mongo-1" should be started with 1 replica And The service "jempi-api" should be connected to the networks | kafka_public | jempi_default | And The service "jempi-postgresql-01" should be started with 1 replica From e0d2d5a20dbb0d312bb3d9c63153ca30c102114f Mon Sep 17 00:00:00 2001 From: drono Date: Wed, 15 May 2024 17:19:41 +0300 Subject: [PATCH 35/43] Platform beta release --- config.yaml | 3 +- fhir-ig-importer/docker-compose.dev.yml | 2 +- .../importer/volume/ig-importer-app.json | 2 +- fhir-ig-importer/package-metadata.json | 4 +- .../package-metadata.json | 4 +- kafka-mapper-consumer/package-metadata.json | 4 +- mpi-mediator/docker-compose.yml | 2 +- .../importer/volume/openhim-import.json | 31 ++---- reprocess-mediator/docker-compose.dev.yml | 2 +- reprocess-mediator/package-metadata.json | 32 +++--- test/cucumber/features/steps/recipesSteps.js | 105 +++++++++++------- 11 files changed, 103 insertions(+), 88 deletions(-) diff --git a/config.yaml b/config.yaml index 54977b16..862a5df6 100644 --- a/config.yaml +++ b/config.yaml @@ -44,6 +44,8 @@ profiles: - openhim-mapping-mediator - kafka-mapper-consumer - kafka-unbundler-consumer + - fhir-ig-importer + - reprocess-mediator envFiles: - cdr-dw.env @@ -75,4 +77,3 @@ profiles: - openhim-mapping-mediator envFiles: - mpi.env - diff --git a/fhir-ig-importer/docker-compose.dev.yml b/fhir-ig-importer/docker-compose.dev.yml index 25714fbd..c33a9169 100644 --- a/fhir-ig-importer/docker-compose.dev.yml +++ b/fhir-ig-importer/docker-compose.dev.yml @@ -10,5 +10,5 @@ services: fhir-ig-importer-ui: ports: - target: 8080 - published: 3000 + published: 3334 mode: host diff --git a/fhir-ig-importer/importer/volume/ig-importer-app.json b/fhir-ig-importer/importer/volume/ig-importer-app.json index a86b0cc8..102b6eac 100644 --- a/fhir-ig-importer/importer/volume/ig-importer-app.json +++ b/fhir-ig-importer/importer/volume/ig-importer-app.json @@ -3,7 +3,7 @@ "description": "FHIR IG microfrontend app", "category": "HIE Configuration", "type": "esmodule", - "url": "http://localhost:3000/jembi-fhir-ig-importer.js", + "url": "http://localhost:3334/jembi-fhir-ig-importer.js", "showInPortal": true, "showInSideBar": true, "access_roles": ["admin"], diff --git a/fhir-ig-importer/package-metadata.json b/fhir-ig-importer/package-metadata.json index 1db85f0a..27f4de82 100644 --- a/fhir-ig-importer/package-metadata.json +++ b/fhir-ig-importer/package-metadata.json @@ -14,7 +14,7 @@ "FHIR_IG_IMPORTER_CORE_PORT": 3001, "FHIR_IG_IMPORTER_CORE_HOST": "0.0.0.0", "FHIR_IG_IMPORTER_CORE_URL": "http://0.0.0.0:3001/fhir/ig/v1.0", - "FHIR_IG_IMPORTER_UI_VERSION": "0.1.0", - "FHIR_IG_IMPORTER_CORE_VERSION": "1.0.0" + "FHIR_IG_IMPORTER_UI_VERSION": "v1.0.0", + "FHIR_IG_IMPORTER_CORE_VERSION": "v1.0.0" } } diff --git a/interoperability-layer-openhim/package-metadata.json b/interoperability-layer-openhim/package-metadata.json index a866f05c..82272031 100644 --- a/interoperability-layer-openhim/package-metadata.json +++ b/interoperability-layer-openhim/package-metadata.json @@ -6,8 +6,8 @@ "type": "infrastructure", "dependencies": [], "environmentVariables": { - "OPENHIM_CORE_IMAGE": "jembi/openhim-core:microfrontends-3.0.0-beta", - "OPENHIM_CONSOLE_IMAGE": "jembi/openhim-console:v1.18.2", + "OPENHIM_CORE_IMAGE": "jembi/openhim-core:v8.4.3", + "OPENHIM_CONSOLE_IMAGE": "jembi/openhim-console:poc-microfrontend", "MONGO_IMAGE": "mongo:4.2", "AWAIT_HELPER_IMAGE": "jembi/await-helper:1.0.1", "MONGO_1_PLACEMENT": "node-1", diff --git a/kafka-mapper-consumer/package-metadata.json b/kafka-mapper-consumer/package-metadata.json index a28cd5da..0c0b17a9 100644 --- a/kafka-mapper-consumer/package-metadata.json +++ b/kafka-mapper-consumer/package-metadata.json @@ -17,7 +17,7 @@ "REGISTER_MEDIATOR": "true", "CLICKHOUSE_HOST": "analytics-datastore-clickhouse", "CLICKHOUSE_PORT": "8123", - "KAFKA_CONSUMER_MAPPER_MEDIATOR_VERSION": "jembi/kafka-mapper-consumer:0.1.0", - "KAFKA_CONSUMER_MAPPER_UI_VERSION": "jembi/kafka-mapper-consumer-ui:0.1.1-alpha" + "KAFKA_CONSUMER_MAPPER_MEDIATOR_VERSION": "jembi/kafka-mapper-consumer:v0.0.1", + "KAFKA_CONSUMER_MAPPER_UI_VERSION": "jembi/kafka-mapper-consumer-ui:v0.0.1" } } diff --git a/mpi-mediator/docker-compose.yml b/mpi-mediator/docker-compose.yml index 084d7e39..8f39f082 100644 --- a/mpi-mediator/docker-compose.yml +++ b/mpi-mediator/docker-compose.yml @@ -2,7 +2,7 @@ version: '3.9' services: mpi-mediator: - image: jembi/mpi-mediator:v2.1.1 + image: jembi/mpi-mediator:v2.2.0 networks: openhim: kafka: diff --git a/mpi-mediator/importer/volume/openhim-import.json b/mpi-mediator/importer/volume/openhim-import.json index 5a1b11b9..cb40af5d 100644 --- a/mpi-mediator/importer/volume/openhim-import.json +++ b/mpi-mediator/importer/volume/openhim-import.json @@ -5,9 +5,7 @@ "surname": "User", "email": "root@openhim.org", "provider": "token", - "groups": [ - "admin" - ], + "groups": ["admin"], "passwordAlgorithm": "sha512", "passwordHash": "ea3824f17cf1379eb118a36bc7c8cf0f45712e2af7748567fca5313dec6fa66d61064e82a5e5cb88e998486ee3c7d0dac235bbeda8c341d6edc1c77406be2ab6", "passwordSalt": "d4f622c0404f09bd959bfb263efa3452", @@ -21,9 +19,7 @@ { "clientID": "test", "name": "Test Client", - "roles": [ - "instant" - ], + "roles": ["instant"], "customTokenID": "test" } ], @@ -98,7 +94,7 @@ "urlPattern": "^/fhir.*$", "methods": ["GET", "POST"], "type": "http", - "priority": null, + "priority": 2, "tcpPort": null, "tcpHost": null, "pollingSchedule": null, @@ -147,9 +143,7 @@ { "name": "MPI Orchestration for fhir bundles - Asynchronous flow", "urlPattern": "^/async/fhir/?$", - "methods": [ - "POST" - ], + "methods": ["POST"], "type": "http", "priority": null, "tcpPort": null, @@ -157,9 +151,7 @@ "pollingSchedule": null, "requestBody": true, "responseBody": true, - "allow": [ - "instant" - ], + "allow": ["instant"], "whitelist": [], "authType": "private", "routes": [ @@ -240,9 +232,7 @@ "pollingSchedule": null, "requestBody": true, "responseBody": true, - "allow": [ - "instant" - ], + "allow": ["instant"], "whitelist": [], "authType": "private", "routes": [ @@ -328,14 +318,9 @@ { "name": "MPI mediator", "urlPattern": "^(/async)?/fhir.*$", - "methods": [ - "POST", - "GET" - ], + "methods": ["POST", "GET"], "type": "http", - "allow": [ - "instant" - ], + "allow": ["instant"], "whitelist": [], "authType": "private", "routes": [ diff --git a/reprocess-mediator/docker-compose.dev.yml b/reprocess-mediator/docker-compose.dev.yml index ee40381a..e376b2f1 100644 --- a/reprocess-mediator/docker-compose.dev.yml +++ b/reprocess-mediator/docker-compose.dev.yml @@ -4,7 +4,7 @@ services: reprocess-mediator: ports: - target: 3000 - published: 3000 + published: 3335 mode: host reprocess-mediator-ui: diff --git a/reprocess-mediator/package-metadata.json b/reprocess-mediator/package-metadata.json index 4393bc6e..97ca4643 100644 --- a/reprocess-mediator/package-metadata.json +++ b/reprocess-mediator/package-metadata.json @@ -1,18 +1,18 @@ { - "id": "reprocess-mediator", - "name": "Reprocess Mediator", - "description": "A mediator that allows for configuring and reprocessing of transactions", - "type": "use-case", - "version": "1.4.2", - "dependencies": ["interoperability-layer-openhim"], - "environmentVariables": { - "TRUST_SELF_SIGNED": "true", - "OPENHIM_MEDIATOR_URL": "https://openhim-core:8080", - "OPENHIM_USERNAME": "root@openhim.org", - "OPENHIM_PASSWORD": "instant101", - "REGISTER_MEDIATOR": "true", - "REPROCESSOR_API_BASE_URL": "http://reprocess-mediator:3000", - "REPROCESS_CORE_VERSION": "latest", - "REPROCESS_UI_VERSION": "latest" - } + "id": "reprocess-mediator", + "name": "Reprocess Mediator", + "description": "A mediator that allows for configuring and reprocessing of transactions", + "type": "use-case", + "version": "1.4.2", + "dependencies": ["interoperability-layer-openhim"], + "environmentVariables": { + "TRUST_SELF_SIGNED": "true", + "OPENHIM_MEDIATOR_URL": "https://openhim-core:8080", + "OPENHIM_USERNAME": "root@openhim.org", + "OPENHIM_PASSWORD": "instant101", + "REGISTER_MEDIATOR": "true", + "REPROCESSOR_API_BASE_URL": "http://reprocess-mediator:3000", + "REPROCESS_CORE_VERSION": "v0.1.0", + "REPROCESS_UI_VERSION": "v0.1.0" } +} diff --git a/test/cucumber/features/steps/recipesSteps.js b/test/cucumber/features/steps/recipesSteps.js index 9c119150..8d24c0b3 100644 --- a/test/cucumber/features/steps/recipesSteps.js +++ b/test/cucumber/features/steps/recipesSteps.js @@ -1,16 +1,15 @@ -"use strict" +"use strict"; const axios = require("axios"); const fs = require("fs"); -const path = require('path'); +const path = require("path"); const chai = require("chai"); -const { ClickHouse } = require('clickhouse'); +const { ClickHouse } = require("clickhouse"); const { Given, When, Then, setDefaultTimeout } = require("@cucumber/cucumber"); setDefaultTimeout(30 * 60 * 1000); -const HOST = - process.env.HOST || 'localhost'; -const CLICKHOUSE_PORT = parseInt(process.env.CLICKHOUSE_PORT || '8124'); +const HOST = process.env.HOST || "localhost"; +const CLICKHOUSE_PORT = parseInt(process.env.CLICKHOUSE_PORT || "8124"); const CLICKHOUSE_DEBUG = Boolean(process.env.CLICKHOUSE_DEBUG || false); const { expect } = chai; @@ -22,69 +21,99 @@ const clickhouse = new ClickHouse({ raw: true, }); -const query = table => `SELECT * FROM ${table}`; +const query = (table) => `SELECT * FROM ${table}`; -const sendRequest = (url, method='POST', data={}) => { +const sendRequest = (url, method = "POST", data = {}) => { return axios({ url, headers: { - 'Content-Type': 'application/fhir+json', - Authorization: `Custom test` + "Content-Type": "application/fhir+json", + Authorization: `Custom test`, }, data, - method - }) + method, + }); }; let PatientID; Given("I have configured the cdr", async function () { await new Promise((resolve) => { - setTimeout(() => resolve(), 300000) + setTimeout(() => resolve(), 300000); }); const organization = JSON.parse( - fs.readFileSync(path.resolve(__dirname, '..' , 'resources', 'organization.json')) + fs.readFileSync( + path.resolve(__dirname, "..", "resources", "organization.json") + ) ); - this.cdrConfigResult = await sendRequest(`http://${HOST}:5001/fhir`, 'POST', organization); + this.cdrConfigResult = await sendRequest( + `http://${HOST}:5001/fhir`, + "POST", + organization + ); }); When("I send a fhir patient bundle", async function () { const fhirBundle = JSON.parse( - fs.readFileSync(path.resolve(__dirname, '..' , 'resources', 'fhirBundle.json')) + fs.readFileSync( + path.resolve(__dirname, "..", "resources", "fhirBundle.json") + ) ); - this.fhirBundleSentResult = await sendRequest(`http://${HOST}:5001/fhir`, 'POST', fhirBundle); + this.fhirBundleSentResult = await sendRequest( + `http://${HOST}:5001/fhir`, + "POST", + fhirBundle + ); }); When("I then send a fhir patient summary request", async function () { - this.IPSResult = await sendRequest(`http://${HOST}:5001/fhir/Patient/${PatientID}/$summary`, 'GET'); -}); - -When("I then send a request for all the patient's clinical data", async function () { - this.EverythingResult = await sendRequest(`http://${HOST}:5001/fhir/Patient/${PatientID}/$everything?_mdm=true`, 'GET'); + this.IPSResult = await sendRequest( + `http://${HOST}:5001/fhir/Patient/${PatientID}/$summary`, + "GET" + ); }); -When("I wait for the services to start up", async function() { +When( + "I then send a request for all the patient's clinical data", + async function () { + this.EverythingResult = await sendRequest( + `http://${HOST}:5001/fhir/Patient/${PatientID}/$everything?_mdm=true`, + "GET" + ); + } +); + +When("I wait for the services to start up", async function () { await new Promise((resolve) => { setTimeout(() => resolve(), 1500000); }); }); Then("the clinical data should be stored in hapi fhir", async function () { - this.fhirBundleSentResult.data.entry.forEach(rec => { + this.fhirBundleSentResult.data.entry.forEach((rec) => { expect(rec.response.status).to.match(/201|200/); }); }); Then("the patient data in the Jempi client registry", async function () { - const patientResource = this.fhirBundleSentResult.data.entry.filter(rec => rec.response.location.match('Patient'))[0]; + const patientResource = this.fhirBundleSentResult.data.entry.filter((rec) => + rec.response.location.match("Patient") + )[0]; - PatientID = patientResource.response.location.split('/')[1]; + PatientID = patientResource.response.location.split("/")[1]; - const patient = await sendRequest(`http://${HOST}:5001/fhir/links/Patient/${PatientID}`, 'GET'); + const patient = await sendRequest( + `http://${HOST}:5001/fhir/Patient/${PatientID}`, + "GET" + ); - expect(patient.data.link.filter(pat => pat.other.reference.match(`Patient/${PatientID}`)).length).to.equal(1); + expect( + patient.data.link.filter((pat) => + pat.other.reference.match(`Patient/${PatientID}`) + ).length + ).to.equal(1); }); Then("I should get a successful summary response", function () { @@ -96,22 +125,22 @@ Then("I should get a successful everything response", function () { }); Then("a request to fetch data from the cdr should fail", async function () { - await sendRequest(`http://${HOST}:5001/fhir/links/Patient/${PatientID}`).catch(err => { - expect(err.message).to.match(/ECONNREFUSED|socket hang up|ETIMEDOUT/); - }); + await sendRequest(`http://${HOST}:5001/fhir/Patient/${PatientID}`).catch( + (err) => { + expect(err.message).to.match(/ECONNREFUSED|socket hang up|ETIMEDOUT/); + } + ); }); Then("the data should be stored in clickhouse", async function () { await new Promise((resolve) => { - setTimeout(() => resolve(), 20000) + setTimeout(() => resolve(), 20000); }); - const patient = await clickhouse.query( - query("patient_example"), - ).toPromise(); - const observation = await clickhouse.query( - query("observation_example") - ).toPromise(); + const patient = await clickhouse.query(query("patient_example")).toPromise(); + const observation = await clickhouse + .query(query("observation_example")) + .toPromise(); expect(JSON.parse(patient).rows).to.be.greaterThan(0); expect(JSON.parse(observation).rows).to.be.greaterThan(0); From 4da745e1e089a430a62c3a2b1aad35e264988643 Mon Sep 17 00:00:00 2001 From: drono Date: Thu, 16 May 2024 06:34:15 +0300 Subject: [PATCH 36/43] Fix failing tests --- fhir-ig-importer/docker-compose.yml | 2 ++ fhir-ig-importer/importer/docker-compose.config.yml | 1 + kafka-mapper-consumer/docker-compose.config.yml | 1 + kafka-mapper-consumer/docker-compose.yml | 7 ++++++- 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/fhir-ig-importer/docker-compose.yml b/fhir-ig-importer/docker-compose.yml index 43c0be1a..1faf0253 100644 --- a/fhir-ig-importer/docker-compose.yml +++ b/fhir-ig-importer/docker-compose.yml @@ -6,6 +6,7 @@ services: networks: hapi-fhir: openhim: + default: environment: HAPI_FHIR_BASE_URL: ${HAPI_FHIR_BASE_URL} HAPI_FHIR_INSTANCES: ${HAPI_FHIR_INSTANCES} @@ -27,4 +28,5 @@ networks: openhim: name: openhim_public external: true + default: diff --git a/fhir-ig-importer/importer/docker-compose.config.yml b/fhir-ig-importer/importer/docker-compose.config.yml index c6398016..0d11921a 100644 --- a/fhir-ig-importer/importer/docker-compose.config.yml +++ b/fhir-ig-importer/importer/docker-compose.config.yml @@ -46,3 +46,4 @@ networks: openhim: name: openhim_public external: true + default: diff --git a/kafka-mapper-consumer/docker-compose.config.yml b/kafka-mapper-consumer/docker-compose.config.yml index 168dd4b8..2e1b4db5 100644 --- a/kafka-mapper-consumer/docker-compose.config.yml +++ b/kafka-mapper-consumer/docker-compose.config.yml @@ -39,3 +39,4 @@ networks: openhim: name: openhim_public external: true + default: diff --git a/kafka-mapper-consumer/docker-compose.yml b/kafka-mapper-consumer/docker-compose.yml index c146a75c..cc031e00 100644 --- a/kafka-mapper-consumer/docker-compose.yml +++ b/kafka-mapper-consumer/docker-compose.yml @@ -21,7 +21,8 @@ services: networks: clickhouse: kafka: - + openhim: + default: kafka-mapper-consumer-ui: image: ${KAFKA_CONSUMER_MAPPER_UI_VERSION} networks: @@ -41,3 +42,7 @@ networks: kafka: name: kafka_public external: true + openhim: + name: openhim_public + external: true + default: From 28902f85f2c55af37b0c29f0194f31e7fe9fcbae Mon Sep 17 00:00:00 2001 From: bradsawadye Date: Thu, 16 May 2024 09:27:55 +0200 Subject: [PATCH 37/43] Add correct assertion THe mpi mediator has been modified and the returned response has a different structure --- test/cucumber/features/steps/recipesSteps.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/test/cucumber/features/steps/recipesSteps.js b/test/cucumber/features/steps/recipesSteps.js index 8d24c0b3..5785eaf4 100644 --- a/test/cucumber/features/steps/recipesSteps.js +++ b/test/cucumber/features/steps/recipesSteps.js @@ -109,11 +109,7 @@ Then("the patient data in the Jempi client registry", async function () { "GET" ); - expect( - patient.data.link.filter((pat) => - pat.other.reference.match(`Patient/${PatientID}`) - ).length - ).to.equal(1); + expect(patient.data.id).to.be.equal(PatientID); }); Then("I should get a successful summary response", function () { From a9687fd52616227660aa6339a65b3685ac5ffd1e Mon Sep 17 00:00:00 2001 From: bradsawadye Date: Thu, 16 May 2024 09:29:39 +0200 Subject: [PATCH 38/43] Ensure that the entry property in the bundle is always an array --- client-registry-jempi/importer/mapping-mediator/searchAll.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client-registry-jempi/importer/mapping-mediator/searchAll.json b/client-registry-jempi/importer/mapping-mediator/searchAll.json index 49134fca..c555e89f 100644 --- a/client-registry-jempi/importer/mapping-mediator/searchAll.json +++ b/client-registry-jempi/importer/mapping-mediator/searchAll.json @@ -9,7 +9,7 @@ }, "inputTransforms": { "total": "$count(lookupRequests.jempiSearchAll.data.goldenRecords)", - "entry": "$map(lookupRequests.jempiSearchAll.data.goldenRecords, function($v) {{'fullUrl': 'Patient/' & $v.goldenId, 'resource': {'resourceType': 'Patient','id': $v.goldenId,'name': {'given': [$v.demographicData.givenName],'family': $v.demographicData.familyName},'address': [{'city': $v.demographicData.city}],'birthDate': $v.demographicData.dob,'telecom': [{'value': $v.demographicData.phoneNumber,'system': 'phone'}],'identifier': [{'system': $v.sourceId.facility,'value': $v.sourceId.patient},{'system': 'NationalID','value': $v.demographicData.nationalId}],'gender': $v.demographicData.gender}}})" + "entry": "$append([], $map(lookupRequests.jempiSearchAll.data.goldenRecords, function($v) {{'fullUrl': 'Patient/' & $v.goldenId, 'resource': {'resourceType': 'Patient','id': $v.goldenId,'name': {'given': [$v.demographicData.givenName],'family': $v.demographicData.familyName},'address': [{'city': $v.demographicData.city}],'birthDate': $v.demographicData.dob,'telecom': [{'value': $v.demographicData.phoneNumber,'system': 'phone'}],'identifier': [{'system': $v.sourceId.facility,'value': $v.sourceId.patient},{'system': 'NationalID','value': $v.demographicData.nationalId}],'gender': $v.demographicData.gender}}}))" }, "inputMapping": { "constants.resourceType": "resourceType", From 29ec53058a2c4cdff108cb81d70db4f1e88a9050 Mon Sep 17 00:00:00 2001 From: drono Date: Thu, 16 May 2024 10:56:50 +0300 Subject: [PATCH 39/43] Fix kafka consumer failing test --- test/cucumber/features/single-mode/kafka-packages.feature | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/cucumber/features/single-mode/kafka-packages.feature b/test/cucumber/features/single-mode/kafka-packages.feature index c37b8f08..6b398d79 100644 --- a/test/cucumber/features/single-mode/kafka-packages.feature +++ b/test/cucumber/features/single-mode/kafka-packages.feature @@ -18,11 +18,11 @@ Feature: Kafka and its dependent packages? And There should be 1 volumes Scenario: Init Kafka Mapper Consumer - Given I use parameters "package init -n=kafka-mapper-consumer --only --dev --env-file=.env.local" + Given I use parameters "package init -n=interoperability-layer-openhim,kafka-mapper-consumer --only --dev --env-file=.env.local" When I launch the platform with params Then The service "kafka-mapper-consumer" should be started with 1 replica And The service "kafka-mapper-consumer" should be connected to the networks - | clickhouse_public | kafka_public | + | clickhouse_public | kafka_public | openhim_public | Scenario: Init Message Bus Kafka Given I use parameters "package init -n=kafka-unbundler-consumer --only --dev --env-file=.env.local" @@ -32,7 +32,7 @@ Feature: Kafka and its dependent packages? | kafka_public | Scenario: Destroy Kafka and its dependent packages - Given I use parameters "package destroy -n=kafka-mapper-consumer,kafka-unbundler-consumer --dev --env-file=.env.local" + Given I use parameters "package destroy -n=kafka-mapper-consumer,kafka-unbundler-consumer,interoperability-layer-openhim --dev --env-file=.env.local" When I launch the platform with params And The service "kafka-01" should be removed And The service "kafdrop" should be removed @@ -43,4 +43,4 @@ Feature: Kafka and its dependent packages? And There should be 0 volume And There should be 0 config And There should not be network - | kafka_public | clickhouse_public | prometheus_public | + | kafka_public | clickhouse_public | prometheus_public | openhim_public | From fa9ad5bf1426b0292bf1ff2a6aea4a39104d5095 Mon Sep 17 00:00:00 2001 From: Ryan Crichton Date: Thu, 16 May 2024 15:33:52 +0200 Subject: [PATCH 40/43] Bump instant version to latest --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 87777566..8667516a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM openhie/package-base:2.3.1 +FROM openhie/package-base:2.3.2 # Install yq RUN curl -L https://github.com/mikefarah/yq/releases/download/v4.23.1/yq_linux_amd64 -o /usr/bin/yq From 56abcb3688416207d00976c555f8ea0f8911a1dd Mon Sep 17 00:00:00 2001 From: bradsawadye Date: Fri, 17 May 2024 11:42:48 +0200 Subject: [PATCH 41/43] Add jempi network for the services to communicate This also adds bootstrapper node placement. Has to be on the leader where the commands are run as we run a command in the bootstrapper docker container --- client-registry-jempi/docker-compose.api.yml | 5 +++++ .../docker-compose.combined-cluster.yml | 17 ++++++++++++++++- .../docker-compose.combined.yml | 12 +++++++++++- .../docker-compose.dgraph-zero-cluster.yml | 13 ++++++++++++- .../docker-compose.dgraph-zero.yml | 7 +++++++ client-registry-jempi/docker-compose.dgraph.yml | 13 +++++++++++++ client-registry-jempi/swarm.sh | 11 +++++------ 7 files changed, 69 insertions(+), 9 deletions(-) diff --git a/client-registry-jempi/docker-compose.api.yml b/client-registry-jempi/docker-compose.api.yml index 4cf156de..7d218748 100644 --- a/client-registry-jempi/docker-compose.api.yml +++ b/client-registry-jempi/docker-compose.api.yml @@ -33,6 +33,7 @@ services: reverse-proxy: kafka: default: + jempi: jempi-api-kc: @@ -77,6 +78,7 @@ services: reverse-proxy: kafka: default: + jempi: volumes: @@ -87,6 +89,9 @@ networks: reverse-proxy: name: reverse-proxy_public external: true + jempi: + name: jempi_public + external: true kafka: name: kafka_public external: true diff --git a/client-registry-jempi/docker-compose.combined-cluster.yml b/client-registry-jempi/docker-compose.combined-cluster.yml index ea3bd37d..2781e809 100644 --- a/client-registry-jempi/docker-compose.combined-cluster.yml +++ b/client-registry-jempi/docker-compose.combined-cluster.yml @@ -6,7 +6,13 @@ services: placement: constraints: - "node.labels.name==node-1" - + + jempi-bootstrapper: + deploy: + placement: + constraints: + - "node.labels.name==node-1" + jempi-postgresql-02: image: bitnami/postgresql-repmgr:15.2.0 environment: @@ -36,6 +42,8 @@ services: configs: - target: /docker-entrypoint-initdb.d/jempi_psql_init_db.sql source: jempi_psql_init_db.sql + networks: + jempi: jempi-postgresql-03: image: bitnami/postgresql-repmgr:15.2.0 @@ -66,7 +74,14 @@ services: configs: - target: /docker-entrypoint-initdb.d/jempi_psql_init_db.sql source: jempi_psql_init_db.sql + networks: + jempi: volumes: jempi-psql-02-data: jempi-psql-03-data: + +networks: + jempi: + name: jempi_public + external: true diff --git a/client-registry-jempi/docker-compose.combined.yml b/client-registry-jempi/docker-compose.combined.yml index 3bca3f1e..0e6a1a2b 100644 --- a/client-registry-jempi/docker-compose.combined.yml +++ b/client-registry-jempi/docker-compose.combined.yml @@ -19,6 +19,7 @@ services: networks: kafka: default: + jempi: jempi-etl: @@ -37,6 +38,7 @@ services: networks: kafka: default: + jempi: jempi-controller: @@ -70,6 +72,7 @@ services: networks: kafka: default: + jempi: jempi-linker: @@ -104,6 +107,7 @@ services: networks: kafka: default: + jempi: jempi-bootstrapper: image: jembi/jempi-bootstrapper:${JEMPI_BOOTSTRAPPER_IMAGE_TAG} @@ -124,6 +128,7 @@ services: networks: kafka: default: + jempi: jempi-postgresql-01: image: bitnami/postgresql-repmgr:15.2.0 @@ -151,6 +156,8 @@ services: configs: - target: /docker-entrypoint-initdb.d/jempi_psql_init_db.sql source: jempi_psql_init_db.sql + networks: + jempi: volumes: jempi-psql-01-data: @@ -161,7 +168,10 @@ networks: kafka: name: kafka_public external: true - defualt: + default: + jempi: + name: jempi_public + external: true configs: diff --git a/client-registry-jempi/docker-compose.dgraph-zero-cluster.yml b/client-registry-jempi/docker-compose.dgraph-zero-cluster.yml index 853f7c7f..fe7e61f8 100644 --- a/client-registry-jempi/docker-compose.dgraph-zero-cluster.yml +++ b/client-registry-jempi/docker-compose.dgraph-zero-cluster.yml @@ -7,7 +7,9 @@ services: constraints: - node.labels.name == node-1 command: dgraph zero --my=jempi-zero-01:5080 --replicas 3 --bindall --raft "idx=1" - + networks: + jempi: + jempi-zero-02: image: dgraph/dgraph:v22.0.0 volumes: @@ -25,6 +27,8 @@ services: restart_policy: condition: on-failure command: dgraph zero --my=jempi-zero-02:5080 --replicas 3 --peer=jempi-zero-01:5080 --raft "idx=2" + networks: + jempi: jempi-zero-03: image: dgraph/dgraph:v22.0.0 @@ -43,7 +47,14 @@ services: restart_policy: condition: on-failure command: dgraph zero --my=jempi-zero-03:5080 --replicas 3 --peer=jempi-zero-01:5080 --raft "idx=3" + networks: + jempi: volumes: jempi-zero-02-data: jempi-zero-03-data: + +networks: + jempi: + name: jempi_public + external: true diff --git a/client-registry-jempi/docker-compose.dgraph-zero.yml b/client-registry-jempi/docker-compose.dgraph-zero.yml index ac1974e3..8b8da6d0 100644 --- a/client-registry-jempi/docker-compose.dgraph-zero.yml +++ b/client-registry-jempi/docker-compose.dgraph-zero.yml @@ -15,6 +15,13 @@ services: restart_policy: condition: on-failure command: dgraph zero --my=jempi-zero-01:5080 --replicas 1 + networks: + jempi: volumes: jempi-zero-01-data: + +networks: + jempi: + name: jempi_public + external: true diff --git a/client-registry-jempi/docker-compose.dgraph.yml b/client-registry-jempi/docker-compose.dgraph.yml index 296aff7d..93ada3f7 100644 --- a/client-registry-jempi/docker-compose.dgraph.yml +++ b/client-registry-jempi/docker-compose.dgraph.yml @@ -16,6 +16,8 @@ services: restart_policy: condition: on-failure command: dgraph alpha --my=jempi-alpha-01:7080 --zero=jempi-zero-01:5080 --security whitelist=0.0.0.0/0 --telemetry "sentry=false;" + networks: + jempi: jempi-alpha-02: image: dgraph/dgraph:v22.0.0 @@ -32,6 +34,8 @@ services: restart_policy: condition: on-failure command: dgraph alpha --my=jempi-alpha-02:7081 --zero=jempi-zero-01:5080 --security whitelist=0.0.0.0/0 -o 1 --telemetry "sentry=false;" + networks: + jempi: jempi-alpha-03: image: dgraph/dgraph:v22.0.0 @@ -48,6 +52,8 @@ services: restart_policy: condition: on-failure command: dgraph alpha --my=jempi-alpha-03:7082 --zero=jempi-zero-01:5080 --security whitelist=0.0.0.0/0 -o 2 --telemetry "sentry=false;" + networks: + jempi: jempi-ratel: image: dgraph/ratel:v21.03.2 @@ -61,8 +67,15 @@ services: restart_policy: condition: on-failure command: dgraph-ratel + networks: + jempi: volumes: jempi-alpha-01-data: jempi-alpha-02-data: jempi-alpha-03-data: + +networks: + jempi: + name: jempi_public + external: true diff --git a/client-registry-jempi/swarm.sh b/client-registry-jempi/swarm.sh index 92e57aa2..d1323691 100644 --- a/client-registry-jempi/swarm.sh +++ b/client-registry-jempi/swarm.sh @@ -52,12 +52,11 @@ function initialize_package() { log info "Running package in PROD mode" fi - # Jempi not working in clustered mode, temporarily disable - # if [[ "$CLUSTERED_MODE" == "true" ]]; then - # dgraph_cluster_compose_param="docker-compose.dgraph-cluster.yml" - # dgraph_zero_cluster_compose_param="docker-compose.dgraph-zero-cluster.yml" - # combined_cluster_compose_param="docker-compose.combined-cluster.yml" - # fi + if [[ "$CLUSTERED_MODE" == "true" ]]; then + dgraph_cluster_compose_param="docker-compose.dgraph-cluster.yml" + dgraph_zero_cluster_compose_param="docker-compose.dgraph-zero-cluster.yml" + combined_cluster_compose_param="docker-compose.combined-cluster.yml" + fi ( log info "Importing JeMPI Kafka topics" From 1cac0f7a79031b1e3489f2cd99776dd5c73fd489 Mon Sep 17 00:00:00 2001 From: drono Date: Mon, 20 May 2024 08:02:51 +0300 Subject: [PATCH 42/43] fix version and disable cert verification in console. --- config.yaml | 2 +- interoperability-layer-openhim/docker-compose.yml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/config.yaml b/config.yaml index 862a5df6..638db7b1 100644 --- a/config.yaml +++ b/config.yaml @@ -1,5 +1,5 @@ projectName: platform -image: jembi/platform:3.0.0-beta +image: jembi/platform:latest logPath: /tmp/logs packages: diff --git a/interoperability-layer-openhim/docker-compose.yml b/interoperability-layer-openhim/docker-compose.yml index 518d93dc..1413642a 100644 --- a/interoperability-layer-openhim/docker-compose.yml +++ b/interoperability-layer-openhim/docker-compose.yml @@ -50,6 +50,8 @@ services: KC_REALM_NAME: ${KC_REALM_NAME} KC_FRONTEND_URL: ${KC_FRONTEND_URL} OPENHIM_CONSOLE_SHOW_LOGIN: ${OPENHIM_CONSOLE_SHOW_LOGIN} + REACT_APP_OPENHIM_API_BASE_URL: ${OPENHIM_API_BASE_URL} + NODE_TLS_REJECT_UNAUTHORIZED: 0 networks: reverse-proxy: keycloak: From 370bd14936c15a2f4e999ce06704e2962901e925 Mon Sep 17 00:00:00 2001 From: bradsawadye Date: Tue, 21 May 2024 12:08:13 +0200 Subject: [PATCH 43/43] Update version for mpi mediator --- mpi-mediator/docker-compose.yml | 2 +- mpi-mediator/package-metadata.json | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/mpi-mediator/docker-compose.yml b/mpi-mediator/docker-compose.yml index 8f39f082..9497ecf8 100644 --- a/mpi-mediator/docker-compose.yml +++ b/mpi-mediator/docker-compose.yml @@ -2,7 +2,7 @@ version: '3.9' services: mpi-mediator: - image: jembi/mpi-mediator:v2.2.0 + image: jembi/mpi-mediator:${MPI_MEDIATOR_VERSION} networks: openhim: kafka: diff --git a/mpi-mediator/package-metadata.json b/mpi-mediator/package-metadata.json index 85c1f07e..ce30f0a9 100644 --- a/mpi-mediator/package-metadata.json +++ b/mpi-mediator/package-metadata.json @@ -7,6 +7,7 @@ "dependencies": ["fhir-datastore-hapi-fhir", "client-registry-jempi"], "environmentVariables": { "MPI_MEDIATOR_INSTANCES": 1, + "MPI_MEDIATOR_VERSION": "v2.3.0", "OPENHIM_MEDIATOR_URL": "https://openhim-core:8080", "TRUST_SELF_SIGNED": "true", "OPENHIM_USERNAME": "root@openhim.org",