diff --git a/client-registry-jempi/config/config-api.json b/client-registry-jempi/config/config-api.json new file mode 100644 index 00000000..dbd6f86a --- /dev/null +++ b/client-registry-jempi/config/config-api.json @@ -0,0 +1,239 @@ +{ + "fields": [ + { + "fieldName": "aux_id", + "fieldType": "String", + "fieldLabel": "AUX ID", + "groups": ["identifiers", "record_details"], + "scope": [ + "/patient-record/:uid", + "/golden-record/:uid", + "/record-details/:uid", + "/search/custom" + ], + "readOnly": true, + "accessLevel": [] + }, + { + "fieldName": "givenName", + "fieldType": "String", + "fieldLabel": "First Name", + "groups": [ + "name", + "demographics", + "filter", + "linked_records", + "record_details" + ], + "scope": [ + "/notifications/match-details", + "/record-details/:uid/relink", + "/search/simple", + "/search/custom", + "/search-results/golden", + "/search-results/patient", + "/record-details/:uid", + "/browse-records" + ], + "validation": { + "required": true, + "onErrorMessage": "The family name cannot be empty" + }, + "accessLevel": [] + }, + { + "fieldName": "familyName", + "fieldType": "String", + "fieldLabel": "Last Name", + "groups": [ + "name", + "demographics", + "filter", + "linked_records", + "record_details" + ], + "scope": [ + "/record-details/:uid", + "/notifications/match-details", + "/record-details/:uid/relink", + "/search/simple", + "/search/custom", + "/search-results/golden", + "/search-results/patient", + "/browse-records" + ], + "validation": { + "required": true, + "onErrorMessage": "The family name cannot be empty" + }, + "accessLevel": [] + }, + { + "fieldName": "gender", + "fieldType": "String", + "fieldLabel": "Gender", + "groups": [ + "demographics", + "filter", + "sub_heading", + "linked_records", + "record_details" + ], + "scope": [ + "/record-details/:uid", + "/notifications/match-details", + "/record-details/:uid/relink", + "/search/custom", + "/browse-records" + ], + "validation": { + "required": true, + "regex": "^(male|female)$", + "onErrorMessage": "The Gender cannot be empty and should either be male, female or neutral" + }, + "accessLevel": [] + }, + { + "fieldName": "dob", + "fieldType": "Date", + "fieldLabel": "Date of Birth", + "groups": [ + "demographics", + "filter", + "sub_heading", + "linked_records", + "record_details" + ], + "scope": [ + "/record-details/:uid", + "/notifications/match-details", + "/record-details/:uid/relink", + "/search/simple", + "/search/custom", + "/search-results/golden", + "/search-results/patient", + "/browse-records" + ], + "accessLevel": [], + "validation": { + "required": true, + "onErrorMessage": "Date of birth cannot be empty" + } + }, + { + "fieldName": "city", + "fieldType": "String", + "fieldLabel": "City", + "groups": ["demographics", "filter", "linked_records", "record_details"], + "scope": [ + "/record-details/:uid", + "/notifications/match-details", + "/record-details/:uid/relink", + "/search/custom", + "/browse-records" + ], + "accessLevel": [], + "validation": { + "required": true, + "onErrorMessage": "Date of birth cannot be empty" + } + }, + { + "fieldName": "phoneNumber", + "fieldType": "String", + "fieldLabel": "Phone No", + "groups": ["demographics", "filter", "linked_records", "record_details"], + "scope": [ + "/record-details/:uid", + "/notifications/match-details", + "/record-details/:uid/relink", + "/search/custom", + "/browse-records" + ], + "accessLevel": [] + }, + { + "fieldName": "nationalId", + "fieldType": "String", + "fieldLabel": "National ID", + "groups": ["identifiers", "linked_records", "record_details"], + "scope": [ + "/record-details/:uid", + "/notifications/match-details", + "/record-details/:uid/relink", + "/search/simple", + "/search/custom", + "/search-results/golden", + "/search-results/patient", + "/browse-records" + ], + "accessLevel": [], + "validation": { + "required": true, + "regex": "", + "onErrorMessage": "The national Id cannot be empty" + } + } + ], + "systemFields": [ + { + "fieldName": "recordType", + "fieldType": "String", + "fieldLabel": "Record Type", + "groups": ["none"], + "scope": ["/notifications/match-details", "/record-details/:uid/relink"], + "accessLevel": [] + }, + { + "fieldName": "uid", + "fieldType": "String", + "fieldLabel": "UID", + "groups": [ + "identifiers", + "sub_heading", + "linked_records", + "record_details", + "filter" + ], + "scope": [ + "/notifications/match-details", + "/record-details/:uid/relink", + "/search-results/golden", + "/search-results/patient", + "/record-details/:uid", + "/browse-records" + ], + "accessLevel": [] + }, + { + "fieldName": "createdAt", + "fieldType": "String", + "fieldLabel": "Created At", + "groups": ["linked_records", "record_details", "audit_trail"], + "scope": ["/record-details/:uid", "/browse-records"], + "accessLevel": [] + }, + { + "fieldName": "sourceId", + "fieldType": "SourceId", + "fieldLabel": "Source Id", + "groups": ["registering_facility", "linked_records", "record_details"], + "scope": ["/record-details/:uid", "/browse-records"], + "accessLevel": [] + }, + { + "fieldName": "score", + "fieldType": "Number", + "fieldLabel": "Score", + "groups": ["none", "record_details"], + "scope": [ + "/patient-record/:uid", + "/golden-record/:uid", + "/record-details/:uid", + "/notifications/match-details", + "/record-details/:uid/relink" + ], + "accessLevel": [] + } + ] +} diff --git a/client-registry-jempi/config/config.json b/client-registry-jempi/config/config.json new file mode 100644 index 00000000..395354cf --- /dev/null +++ b/client-registry-jempi/config/config.json @@ -0,0 +1,189 @@ +{ + "auxInteractionFields": [ + { + "fieldName": "aux_date_created", + "fieldType": "DateTime" + }, + { + "fieldName": "aux_id", + "fieldType": "String", + "source": { + "csvCol": 0 + } + }, + { + "fieldName": "aux_clinical_data", + "fieldType": "String", + "source": { + "csvCol": 10 + } + } + ], + "auxGoldenRecordFields": [ + { + "fieldName": "aux_date_created", + "fieldType": "DateTime" + }, + { + "fieldName": "aux_auto_update_enabled", + "fieldType": "Bool", + "default": "true" + }, + { + "fieldName": "aux_id", + "fieldType": "String", + "source": { + "interactionField": "aux_id" + } + } + ], + "additionalNodes": [ + { + "nodeName": "SourceId", + "fields": [ + { + "fieldName": "facility", + "fieldType": "String", + "source": { + "csvCol": 8 + } + }, + { + "fieldName": "patient", + "fieldType": "String", + "source": { + "csvCol": 9 + } + } + ] + } + ], + "demographicFields": [ + { + "fieldName": "given_name", + "fieldType": "String", + "source": { + "csvCol": 1 + }, + "indexGoldenRecord": "@index(exact,trigram)", + "indexInteraction": "@index(exact,trigram)", + "linkMetaData": { + "comparison": "JARO_WINKLER_SIMILARITY", + "comparisonLevels": [0.92], + "m": 0.8806329, + "u": 0.0026558 + } + }, + { + "fieldName": "family_name", + "fieldType": "String", + "source": { + "csvCol": 2 + }, + "indexGoldenRecord": "@index(exact,trigram)", + "indexInteraction": "@index(exact,trigram)", + "linkMetaData": { + "comparison": "JARO_WINKLER_SIMILARITY", + "comparisonLevels": [0.92], + "m": 0.9140443, + "u": 0.0006275 + } + }, + { + "fieldName": "gender", + "fieldType": "String", + "source": { + "csvCol": 3 + }, + "indexGoldenRecord": "@index(exact,trigram)", + "linkMetaData": { + "comparison": "JARO_WINKLER_SIMILARITY", + "comparisonLevels": [0.92], + "m": 0.9468393, + "u": 0.4436446 + } + }, + { + "fieldName": "dob", + "fieldType": "String", + "source": { + "csvCol": 4 + }, + "linkMetaData": { + "comparison": "JARO_WINKLER_SIMILARITY", + "comparisonLevels": [0.92], + "m": 0.7856196, + "u": 0.0000465 + } + }, + { + "fieldName": "city", + "fieldType": "String", + "source": { + "csvCol": 5 + }, + "indexGoldenRecord": "@index(trigram)", + "linkMetaData": { + "comparison": "JARO_WINKLER_SIMILARITY", + "comparisonLevels": [0.92], + "m": 0.8445694, + "u": 0.0355741 + } + }, + { + "fieldName": "phone_number", + "fieldType": "String", + "source": { + "csvCol": 6 + }, + "indexGoldenRecord": "@index(exact,trigram)", + "linkMetaData": { + "comparison": "JARO_WINKLER_SIMILARITY", + "comparisonLevels": [0.92], + "m": 0.84085, + "u": 0.0000004 + } + }, + { + "fieldName": "national_id", + "fieldType": "String", + "source": { + "csvCol": 7 + }, + "indexGoldenRecord": "@index(exact,trigram)", + "indexInteraction": "@index(exact,trigram)", + "linkMetaData": { + "comparison": "JARO_WINKLER_SIMILARITY", + "comparisonLevels": [0.92], + "m": 0.8441029, + "u": 0.0000002 + } + } + ], + "rules": { + "link": { + "deterministic": [ + { + "vars": ["national_id"], + "text": "eq(national_id)" + }, + { + "vars": ["given_name", "family_name", "phone_number"], + "text": "eq(given_name) and eq(family_name) and eq(phone_number)" + } + ], + "probabilistic": [ + { + "vars": [ + "given_name", + "family_name", + "city", + "phone_number", + "national_id" + ], + "text": "match(given_name,3) and match(family_name,3) or match(given_name,3) and match(city,3) or match(family_name,3) and match(city,3) or match(phone_number,2) or match(national_id,3)" + } + ] + } + } +} diff --git a/client-registry-jempi/docker-compose.api.yml b/client-registry-jempi/docker-compose.api.yml index e216624a..d586f930 100644 --- a/client-registry-jempi/docker-compose.api.yml +++ b/client-registry-jempi/docker-compose.api.yml @@ -3,6 +3,11 @@ version: "3.9" services: jempi-api: image: jembi/jempi-api:${JEMPI_API_IMAGE_TAG} + configs: + - target: /app/conf_system/config.json + source: config.json + - target: /app/conf_system/config-api.json + source: config-api.json environment: LOG4J2_LEVEL: ${LOG4J2_LEVEL} POSTGRESQL_IP: ${JEMPI_POSTGRES_DB} @@ -20,6 +25,11 @@ services: LINKER_HTTP_PORT: 50000 CONTROLLER_IP: jempi-controller CONTROLLER_HTTP_PORT: 50000 + SYSTEM_CONFIG_DIR: ${SYSTEM_CONFIG_DIR} + API_CONFIG_REFERENCE_FILENAME: ${API_CONFIG_REFERENCE_FILENAME} + API_CONFIG_MASTER_FILENAME: ${API_CONFIG_MASTER_FILENAME} + API_FIELDS_CONFIG_FILENAME: ${API_FIELDS_CONFIG_FILENAME} + JEMPI_FILE_IMPORT_MAX_SIZE_BYTE: ${JEMPI_FILE_IMPORT_MAX_SIZE_BYTE} volumes: - "jempi-shared-data:/app/csv" deploy: @@ -45,6 +55,11 @@ services: jempi-api-kc: image: jembi/jempi-api-kc:${JEMPI_API_KC_IMAGE_TAG} + configs: + - target: /app/conf_system/config.json + source: config.json + - target: /app/conf_system/config-api.json + source: config-api.json environment: LOG4J2_LEVEL: ${LOG4J2_LEVEL} KC_REALM_NAME: ${KC_REALM_NAME} @@ -72,6 +87,11 @@ services: LINKER_HTTP_PORT: 50000 CONTROLLER_IP: jempi-controller CONTROLLER_HTTP_PORT: 50000 + SYSTEM_CONFIG_DIR: ${SYSTEM_CONFIG_DIR} + API_CONFIG_REFERENCE_FILENAME: ${API_CONFIG_REFERENCE_FILENAME} + API_CONFIG_MASTER_FILENAME: ${API_CONFIG_MASTER_FILENAME} + API_FIELDS_CONFIG_FILENAME: ${API_FIELDS_CONFIG_FILENAME} + JEMPI_FILE_IMPORT_MAX_SIZE_BYTE: ${JEMPI_FILE_IMPORT_MAX_SIZE_BYTE} volumes: - "jempi-shared-data:/app/csv" deploy: @@ -109,3 +129,15 @@ networks: name: kafka_public external: true default: + +configs: + config.json: + file: ./config/config.json + name: config.json-${config_json_DIGEST:?err} + labels: + name: jempi + config-api.json: + file: ./config/config-api.json + name: config-api.json-${config_api_json_DIGEST:?err} + labels: + name: jempi diff --git a/client-registry-jempi/docker-compose.combined-dev.yml b/client-registry-jempi/docker-compose.combined-dev.yml deleted file mode 100644 index 628aab66..00000000 --- a/client-registry-jempi/docker-compose.combined-dev.yml +++ /dev/null @@ -1,16 +0,0 @@ -version: '3.9' - -services: - jempi-controller: - ports: - - published: 50020 - target: 50000 - protocol: tcp - mode: host - - jempi-linker: - ports: - - published: 50010 - target: 50000 - protocol: tcp - mode: host diff --git a/client-registry-jempi/docker-compose.combined.yml b/client-registry-jempi/docker-compose.combined.yml index 230fee87..9822c7c8 100644 --- a/client-registry-jempi/docker-compose.combined.yml +++ b/client-registry-jempi/docker-compose.combined.yml @@ -3,6 +3,9 @@ version: "3.9" services: jempi-async-receiver: image: jembi/jempi-async-receiver:${JEMPI_ASYNC_RECEIVER_IMAGE_TAG} + configs: + - target: /app/conf_system/config.json + source: config.json environment: LOG4J2_LEVEL: ${LOG4J2_LEVEL} KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_HOSTS} @@ -23,6 +26,9 @@ services: jempi-etl: image: jembi/jempi-etl:${JEMPI_ETL_IMAGE_TAG} + configs: + - target: /app/conf_system/config.json + source: config.json environment: LOG4J2_LEVEL: ${LOG4J2_LEVEL} KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_HOSTS} @@ -41,6 +47,9 @@ services: jempi-controller: image: jembi/jempi-controller:${JEMPI_CONTROLLER_IMAGE_TAG} + configs: + - target: /app/conf_system/config.json + source: config.json environment: LOG4J2_LEVEL: ${LOG4J2_LEVEL} POSTGRESQL_IP: ${JEMPI_POSTGRES_DB} @@ -72,9 +81,32 @@ services: default: jempi: postgres: + + jempi-em-scala: + image: jembi/jempi-em-scala:${JEMPI_EM_SCALA_IMAGE_TAG} + configs: + - target: /app/conf_system/config.json + source: config.json + environment: + LOG4J2_LEVEL: ${LOG4J2_LEVEL} + KAFKA_SERVERS: ${KAFKA_HOSTS} + deploy: + replicas: ${JEMPI_EM_SCALA_INSTANCES} + resources: + limits: + memory: ${JEMPI_EM_SCALA_MEMORY_LIMIT} + reservations: + memory: ${JEMPI_EM_SCALA_MEMORY_RESERVE} + networks: + kafka: + default: + jempi: jempi-linker: image: jembi/jempi-linker:${JEMPI_LINKER_IMAGE_TAG} + configs: + - target: /app/conf_system/config.json + source: config.json environment: LOG4J2_LEVEL: ${LOG4J2_LEVEL} POSTGRESQL_IP: ${JEMPI_POSTGRES_DB} @@ -110,6 +142,9 @@ services: jempi-bootstrapper: image: jembi/jempi-bootstrapper:${JEMPI_BOOTSTRAPPER_IMAGE_TAG} + configs: + - target: /app/conf_system/config.json + source: config.json environment: POSTGRESQL_IP: ${POSTGRES_SERVICE} POSTGRESQL_PORT: 5432 @@ -144,3 +179,10 @@ networks: postgres: name: postgres_public external: true + +configs: + config.json: + file: ./config/config.json + name: config.json-${config_json_DIGEST:?err} + labels: + name: jempi diff --git a/client-registry-jempi/docker-compose.dgraph-cluster.yml b/client-registry-jempi/docker-compose.dgraph-cluster.yml index ed1b9305..03ede30b 100644 --- a/client-registry-jempi/docker-compose.dgraph-cluster.yml +++ b/client-registry-jempi/docker-compose.dgraph-cluster.yml @@ -6,21 +6,60 @@ services: placement: constraints: - node.labels.name == node-1 - + jempi-alpha-02: + image: ${JEMPI_ZERO_IMAGE} + hostname: alpha-02 + volumes: + - jempi-alpha-02-data:/dgraph deploy: placement: constraints: - node.labels.name == node-2 + replicas: 1 + resources: + limits: + memory: ${JEMPI_ALPHA_02_MEMORY_LIMIT} + reservations: + memory: ${JEMPI_ALPHA_02_MEMORY_RESERVE} + restart_policy: + condition: on-failure + command: dgraph alpha --my=jempi-alpha-02:7081 --zero=jempi-zero-01:5080 --security whitelist=0.0.0.0/0 -o 1 --telemetry "sentry=false;" + networks: + jempi: jempi-alpha-03: + image: ${JEMPI_ZERO_IMAGE} + hostname: alpha-03 + volumes: + - jempi-alpha-03-data:/dgraph deploy: placement: - constraints: - - node.labels.name == node-3 + constraints: + - node.labels.name == node-3 + replicas: 1 + resources: + limits: + memory: ${JEMPI_ALPHA_03_MEMORY_LIMIT} + reservations: + memory: ${JEMPI_ALPHA_03_MEMORY_RESERVE} + restart_policy: + condition: on-failure + command: dgraph alpha --my=jempi-alpha-03:7082 --zero=jempi-zero-01:5080 --security whitelist=0.0.0.0/0 -o 2 --telemetry "sentry=false;" + networks: + jempi: jempi-ratel: deploy: placement: constraints: - node.labels.name == node-1 + +volumes: + jempi-alpha-02-data: + jempi-alpha-03-data: + +networks: + jempi: + name: jempi_public + external: true diff --git a/client-registry-jempi/docker-compose.dgraph-dev.yml b/client-registry-jempi/docker-compose.dgraph-dev.yml index 51be6bac..9bcdc281 100644 --- a/client-registry-jempi/docker-compose.dgraph-dev.yml +++ b/client-registry-jempi/docker-compose.dgraph-dev.yml @@ -12,28 +12,6 @@ services: protocol: tcp mode: host - jempi-alpha-02: - ports: - - published: 8071 - target: 8081 - protocol: tcp - mode: host - - published: 9081 - target: 9081 - protocol: tcp - mode: host - - jempi-alpha-03: - ports: - - published: 8072 - target: 8082 - protocol: tcp - mode: host - - published: 9082 - target: 9082 - protocol: tcp - mode: host - jempi-ratel: ports: - published: 8010 diff --git a/client-registry-jempi/docker-compose.dgraph.yml b/client-registry-jempi/docker-compose.dgraph.yml index 2f1c30ce..e56f52e0 100644 --- a/client-registry-jempi/docker-compose.dgraph.yml +++ b/client-registry-jempi/docker-compose.dgraph.yml @@ -15,43 +15,7 @@ services: memory: ${JEMPI_ALPHA_01_MEMORY_RESERVE} restart_policy: condition: on-failure - command: dgraph alpha --my=jempi-alpha-01:7080 --zero=jempi-zero-01:5080 --security whitelist=0.0.0.0/0 --telemetry "sentry=false;" - networks: - jempi: - - jempi-alpha-02: - image: ${JEMPI_ZERO_IMAGE} - hostname: alpha-02 - volumes: - - jempi-alpha-02-data:/dgraph - deploy: - replicas: 1 - resources: - limits: - memory: ${JEMPI_ALPHA_02_MEMORY_LIMIT} - reservations: - memory: ${JEMPI_ALPHA_02_MEMORY_RESERVE} - restart_policy: - condition: on-failure - command: dgraph alpha --my=jempi-alpha-02:7081 --zero=jempi-zero-01:5080 --security whitelist=0.0.0.0/0 -o 1 --telemetry "sentry=false;" - networks: - jempi: - - jempi-alpha-03: - image: ${JEMPI_ZERO_IMAGE} - hostname: alpha-03 - volumes: - - jempi-alpha-03-data:/dgraph - deploy: - replicas: 1 - resources: - limits: - memory: ${JEMPI_ALPHA_03_MEMORY_LIMIT} - reservations: - memory: ${JEMPI_ALPHA_03_MEMORY_RESERVE} - restart_policy: - condition: on-failure - command: dgraph alpha --my=jempi-alpha-03:7082 --zero=jempi-zero-01:5080 --security whitelist=0.0.0.0/0 -o 2 --telemetry "sentry=false;" + command: dgraph alpha --my=jempi-alpha-01:7080 --zero=jempi-zero-01:5080 --cache "size-mb=4096; percentage=50,30,20;" --security whitelist=0.0.0.0/0 --telemetry "sentry=false;" networks: jempi: @@ -72,8 +36,6 @@ services: volumes: jempi-alpha-01-data: - jempi-alpha-02-data: - jempi-alpha-03-data: networks: jempi: diff --git a/client-registry-jempi/importer/openhim/openhimConfig.js b/client-registry-jempi/importer/openhim/openhimConfig.js index e1a797c0..37dc1a19 100644 --- a/client-registry-jempi/importer/openhim/openhimConfig.js +++ b/client-registry-jempi/importer/openhim/openhimConfig.js @@ -7,7 +7,7 @@ const path = require("path"); const OPENHIM_CORE_SERVICE_NAME = 'openhim-core' const OPENHIM_MEDIATOR_API_PORT = 8080 const OPENHIM_API_PASSWORD = - process.env.OPENHIM_API_PASSWORD || 'openhim-password' + process.env.OPENHIM_API_PASSWORD || 'instant101' const OPENHIM_API_USERNAME = process.env.OPENHIM_API_USERNAME || 'root@openhim.org' diff --git a/client-registry-jempi/package-metadata.json b/client-registry-jempi/package-metadata.json index 2589a198..6c810725 100644 --- a/client-registry-jempi/package-metadata.json +++ b/client-registry-jempi/package-metadata.json @@ -13,14 +13,14 @@ "environmentVariables": { "KAFKA_HOSTS": "kafka-01:9092", "JEMPI_POSTGRESQL_IMAGE": "bitnami/postgresql-repmgr:15.2.0", - "JEMPI_ZERO_IMAGE": "dgraph/dgraph:v22.0.0", + "JEMPI_ZERO_IMAGE": "dgraph/dgraph:v23.1.1", "JEMPI_RATEL_IMAGE": "dgraph/ratel:v21.03.2", "JEMPI_ZERO_MEMORY_LIMIT": "3G", "JEMPI_ZERO_MEMORY_RESERVE": "500M", "JEMPI_ZERO_01_PLACEMENT": "node-1", "JEMPI_ZERO_02_PLACEMENT": "node-2", "JEMPI_ZERO_03_PLACEMENT": "node-3", - "JEMPI_ALPHA_01_MEMORY_LIMIT": "3G", + "JEMPI_ALPHA_01_MEMORY_LIMIT": "6G", "JEMPI_ALPHA_01_MEMORY_RESERVE": "500M", "JEMPI_ALPHA_02_MEMORY_LIMIT": "3G", "JEMPI_ALPHA_02_MEMORY_RESERVE": "500M", @@ -40,23 +40,23 @@ "JEMPI_ETL_MEMORY_RESERVE": "500M", "JEMPI_API_INSTANCES": 1, "JEMPI_KAFKA_TOPICS": "JeMPI-async-etl,JeMPI-interaction-controller,JeMPI-interaction-em,JeMPI-interaction-linker,JeMPI-mu-linker,JeMPI-audit-trail,JeMPI-notifications", - "JEMPI_ASYNC_RECEIVER_IMAGE_TAG": "1.0.1-beta", - "JEMPI_CONTROLLER_IMAGE_TAG": "1.0.1-beta", - "JEMPI_LINKER_IMAGE_TAG": "1.0.1-beta", - "JEMPI_API_IMAGE_TAG": "1.0.1-beta", - "JEMPI_API_KC_IMAGE_TAG": "1.0.1-beta", - "JEMPI_ETL_IMAGE_TAG": "1.0.1-beta", + "JEMPI_ASYNC_RECEIVER_IMAGE_TAG": "1.1.0", + "JEMPI_CONTROLLER_IMAGE_TAG": "1.1.0", + "JEMPI_LINKER_IMAGE_TAG": "1.1.1", + "JEMPI_API_IMAGE_TAG": "1.1.1", + "JEMPI_API_KC_IMAGE_TAG": "1.1.1", + "JEMPI_ETL_IMAGE_TAG": "1.1.0", "JEMPI_ASYNC_RECEIVER_INSTANCES": 1, "JEMPI_CONTROLLER_INSTANCES": 1, - "JEMPI_LINKER_INSTANCES": 1, + "JEMPI_LINKER_INSTANCES": 3, "JEMPI_ETL_INSTANCES": 1, "JEMPI_OPENHIM_PASSWORD": "instant101", "JEMPI_SESSION_SECRET": "c05ll3lesrinf39t7mc5h6un6r0c69lgfno69dsak3vabeqamouq4328cuaekros401ajdpkh60rrt", - "JEMPI_FILE_IMPORT_MAX_SIZE_BYTE": 128000000, + "JEMPI_FILE_IMPORT_MAX_SIZE_BYTE": "128m", "REACT_APP_JEMPI_BASE_API_HOST": "http://localhost", "REACT_APP_JEMPI_BASE_API_PORT": "50000", "REACT_APP_MOCK_BACKEND": "false", - "JEMPI_WEB_VERSION": "1.0.1-beta", + "JEMPI_WEB_VERSION": "1.1.0", "JEMPI_WEB_INSTANCES": 1, "JEMPI_WEB_MEMORY_LIMIT": "2G", "JEMPI_WEB_MEMORY_RESERVE": "500M", @@ -69,7 +69,7 @@ "KC_JEMPI_ROOT_URL": "http://localhost:3033", "POSTGRES_SERVICE": "postgres-1", "JEMPI_POSTGRES_DB": "postgres-1", - "JEMPI_POSTGRESQL_USER": "jempi", + "JEMPI_POSTGRESQL_USERNAME": "postgres", "JEMPI_POSTGRESQL_PASSWORD": "instant101", "POSTGRESQL_USERS_DB": "users_db", "POSTGRESQL_NOTIFICATIONS_DB": "notifications_db", @@ -82,8 +82,8 @@ "JEMPI_SESSION_DOMAIN_NAME": "localhost", "DOMAIN_NAME": "", "KAFKA_APPLICATION_ID_API": "api-app-id", - "DGRAPH_HOSTS": "jempi-alpha-01,jempi-alpha-02,jempi-alpha-03", - "DGRAPH_PORTS": "9080,9081,9082", + "DGRAPH_HOSTS": "jempi-alpha-01", + "DGRAPH_PORTS": "9080", "LOG4J2_LEVEL": "DEBUG", "KAFKA_APPLICATION_ID_INTERACTIONS": "app-id-link1", "KAFKA_APPLICATION_ID_MU": "app-id-link2", @@ -95,7 +95,15 @@ "KAFKA_APPLICATION_ID_CTRL": "app-id-ctrl", "KAFKA_CLIENT_ID_CTRL": "client-id-ctrl", "KAFKA_APPLICATION_ID_BOOTSTRAPPER": "app-id-bootstrapper", - "JEMPI_BOOTSTRAPPER_IMAGE_TAG": "1.0.1-beta", + "JEMPI_BOOTSTRAPPER_IMAGE_TAG": "1.1.0", + "JEMPI_EM_SCALA_IMAGE_TAG": "dev", + "JEMPI_EM_SCALA_INSTANCES": "1", + "JEMPI_EM_SCALA_MEMORY_LIMIT": "3G", + "JEMPI_EM_SCALA_MEMORY_RESERVE": "500M", + "SYSTEM_CONFIG_DIR": "/app/conf_system", + "API_CONFIG_REFERENCE_FILENAME": "config.json", + "API_CONFIG_MASTER_FILENAME": "config-master.json", + "API_FIELDS_CONFIG_FILENAME": "config-api.json", "JEMPI_API_KC_TRAEFIK_SUBDOMAIN": "jempi-api-kc", "JEMPI_API_TRAEFIK_SUBDOMAIN": "jempi-api", "JEMPI_WEB_TRAEFIK_SUBDOMAIN": "jempi-web" diff --git a/client-registry-jempi/swarm.sh b/client-registry-jempi/swarm.sh index 808bdd51..2504799d 100644 --- a/client-registry-jempi/swarm.sh +++ b/client-registry-jempi/swarm.sh @@ -34,7 +34,6 @@ function import_sources() { function initialize_package() { local dgraph_dev_compose_param="" local dgraph_zero_dev_compose_param="" - local combined_dev_compose_param="" local combined_cluster_compose_param="" local api_dev_compose_param="" local web_dev_compose_param="" @@ -45,7 +44,6 @@ function initialize_package() { log info "Running package in DEV mode" dgraph_dev_compose_param="docker-compose.dgraph-dev.yml" dgraph_zero_dev_compose_param="docker-compose.dgraph-zero-dev.yml" - combined_dev_compose_param="docker-compose.combined-dev.yml" api_dev_compose_param="docker-compose.api-dev.yml" web_dev_compose_param="docker-compose.web-dev.yml" else @@ -74,7 +72,7 @@ function initialize_package() { docker::deploy_service $STACK "${COMPOSE_FILE_PATH}" "docker-compose.dgraph.yml" "$dgraph_dev_compose_param" "$dgraph_cluster_compose_param" log info "Deploy other combined services" - docker::deploy_service $STACK "${COMPOSE_FILE_PATH}" "docker-compose.combined.yml" "$combined_dev_compose_param" "$combined_cluster_compose_param" + docker::deploy_service $STACK "${COMPOSE_FILE_PATH}" "docker-compose.combined.yml" "$combined_cluster_compose_param" log info "Deploy JeMPI API" docker::deploy_service $STACK "${COMPOSE_FILE_PATH}" "docker-compose.api.yml" "$api_dev_compose_param" diff --git a/config.yaml b/config.yaml index c44ed3f8..c422b527 100644 --- a/config.yaml +++ b/config.yaml @@ -82,3 +82,16 @@ profiles: - openhim-mapping-mediator envFiles: - mpi.env + + - name: mpi-qa + packages: + - interoperability-layer-openhim + - reverse-proxy-nginx + - message-bus-kafka + - job-scheduler-ofelia + - monitoring + - client-registry-jempi + - identity-access-manager-keycloak + - openhim-mapping-mediator + envFiles: + - mpi-qa.env diff --git a/interoperability-layer-openhim/package-metadata.json b/interoperability-layer-openhim/package-metadata.json index fb92768a..372bc954 100644 --- a/interoperability-layer-openhim/package-metadata.json +++ b/interoperability-layer-openhim/package-metadata.json @@ -7,7 +7,7 @@ "dependencies": [], "environmentVariables": { "OPENHIM_CORE_IMAGE": "jembi/openhim-core:v8.4.3", - "OPENHIM_CONSOLE_IMAGE": "jembi/openhim-console:poc-microfrontend", + "OPENHIM_CONSOLE_IMAGE": "jembi/openhim-console:v1.18.2", "MONGO_IMAGE": "mongo:4.2", "AWAIT_HELPER_IMAGE": "jembi/await-helper:1.0.1", "MONGO_1_PLACEMENT": "node-1", diff --git a/kafka-mapper-consumer/docker-compose.yml b/kafka-mapper-consumer/docker-compose.yml index 9f98831c..cdd96cda 100644 --- a/kafka-mapper-consumer/docker-compose.yml +++ b/kafka-mapper-consumer/docker-compose.yml @@ -15,6 +15,7 @@ services: REGISTER_MEDIATOR: ${REGISTER_MEDIATOR} CLICKHOUSE_HOST: ${CLICKHOUSE_HOST} CLICKHOUSE_PORT: ${CLICKHOUSE_PORT} + CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD} configs: - target: /app/src/data/fhir-mapping.json source: fhir-mapping.json diff --git a/kafka-mapper-consumer/package-metadata.json b/kafka-mapper-consumer/package-metadata.json index 9fa23ea9..c9f9401b 100644 --- a/kafka-mapper-consumer/package-metadata.json +++ b/kafka-mapper-consumer/package-metadata.json @@ -17,7 +17,8 @@ "REGISTER_MEDIATOR": "true", "CLICKHOUSE_HOST": "analytics-datastore-clickhouse", "CLICKHOUSE_PORT": "8123", - "KAFKA_CONSUMER_MAPPER_MEDIATOR_VERSION": "jembi/kafka-mapper-consumer:latest", + "CLICKHOUSE_PASSWORD": "dev_password_only", + "KAFKA_CONSUMER_MAPPER_MEDIATOR_VERSION": "jembi/kafka-mapper-consumer:new-pass", "KAFKA_CONSUMER_MAPPER_UI_VERSION": "jembi/kafka-mapper-consumer-ui:latest", "OPENHIM_CONSOLE_BASE_URL": "http://localhost:9000" } diff --git a/mpi-qa.env b/mpi-qa.env new file mode 100644 index 00000000..4a883971 --- /dev/null +++ b/mpi-qa.env @@ -0,0 +1,130 @@ +# General + +CLUSTERED_MODE=true + +# Log + +DEBUG=1 +BASHLOG_FILE=0 +BASHLOG_FILE_PATH=platform.log + +# Interoperability Layer - OpenHIM +OPENHIM_CORE_INSTANCES=3 +OPENHIM_CONSOLE_INSTANCES=3 +OPENHIM_CORE_MEDIATOR_HOSTNAME=openhimcomms.jempi-qa.jembi-skunks.org +OPENHIM_MEDIATOR_API_PORT=443 +MONGO_SET_COUNT=3 +OPENHIM_MONGO_URL=mongodb://mongo-1:27017,mongo-2:27017,mongo-3:27017/openhim?replicaSet=mongo-set +OPENHIM_MONGO_ATNAURL=mongodb://mongo-1:27017,mongo-2:27017,mongo-3:27017/openhim?replicaSet=mongo-set +KAFKA_BROKERS=kafka-01,kafka-02,kafka-03 + +# FHIR Datastore - HAPI FHIR +HAPI_FHIR_INSTANCES=3 +# (pool size * instances) should be less than 100 +HF_MAX_POOL_SIZE=30 +REPMGR_PARTNER_NODES=postgres-1,postgres-2,postgres-3 +POSTGRES_REPLICA_SET=postgres-1:5432,postgres-2:5432,postgres-3:5432 +HAPI_DB_SET=pgpool-1:5432,pgpool-2:5432,pgpool-3:5432 + +# Sante Datastore - Sante MPI +SANTEMPI_REPMGR_PARTNER_NODES=santempi-psql-1,santempi-psql-2,santempi-psql-3 +SANTEMPI_DB_REP_SET=pgpool-1:5432,pgpool-2:5432,pgpool-3:5432 + +# Reverse Proxy - Nginx +REVERSE_PROXY_INSTANCES=1 +DOMAIN_NAME=jempi-qa.jembi-skunks.org +SUBDOMAINS=openhimcomms.jempi-qa.jembi-skunks.org,openhimcore.jempi-qa.jembi-skunks.org,openhimconsole.jempi-qa.jembi-skunks.org,kibana.jempi-qa.jembi-skunks.org,reports.jempi-qa.jembi-skunks.org,santewww.jempi-qa.jembi-skunks.org,santempi.jempi-qa.jembi-skunks.org,superset.jempi-qa.jembi-skunks.org,keycloak.jempi-qa.jembi-skunks.org,grafana.jempi-qa.jembi-skunks.org,minio.jempi-qa.jembi-skunks.org,jempi-web.jempi-qa.jembi-skunks.org,jempi-api.jempi-qa.jembi-skunks.org +STAGING=false +INSECURE=false + +# Analytics Datastore - Elastic Search +ES_HEAP_SIZE=-Xms8192m -Xmx8192m +ES_LEADER_NODE=analytics-datastore-elastic-search-01 +ES_HOSTS="\"analytics-datastore-elastic-search-01:9200","analytics-datastore-elastic-search-02:9200","analytics-datastore-elastic-search-03:9200\"" + +# Analytics Datastore - Clickhouse +CLICKHOUSE_HOST=analytics-datastore-clickhouse-01 + +# Dashboard Vusualizer - Superset +SUPERSET_POSTGRESQL_URL=pgpool-1:5432,pgpool-2:5432,pgpool-3:5432 + +# Dashboard Visualiser - Kibana +KIBANA_INSTANCES=1 + +# Data Mapper - Logstash +LS_JAVA_OPTS=-Xmx6144m -Xms6144m +LOGSTASH_INSTANCES=3 +LOGSTASH_DEV_MOUNT=false +LOGSTASH_PACKAGE_PATH= + +# Dashboard Visualiser - JS Reports +JS_REPORT_INSTANCES=1 +JS_REPORT_LICENSE_KEY= + +# MAKE SURE YOU HAVE RUN 'set-permissions.sh' SCRIPT BEFORE AND AFTER RUNNING JS REPORT +JS_REPORT_DEV_MOUNT=false +JS_REPORT_PACKAGE_PATH= + +# Message Bus Kafka +# Topics should comma seperated, optional include partion and repliction values +# e.g. :: -> test:3:2 (defaults to :3:1) +KAFKA_TOPICS=2xx,2xx-async,reprocess,3xx,metrics:3:3,patient,observation +KAFKA_HOSTS=kafka-01:9092,kafka-02:9092,kafka-03:9092 + +# Kafka consumer mapper +KAFKA_HOST=kafka-01,kafka-02,kafka-03 +KAFKA_PORT=9092 + +# MPI MEDIATOR +MPI_MEDIATOR_INSTANCES=3 + +# Message Bus Helper - Hapi Proxy +HAPI_PROXY_INSTANCES=3 + +# Identity Access Manager - Keycloak +KC_FRONTEND_URL=https://keycloak.jempi-qa.jembi-skunks.org +KC_GRAFANA_ROOT_URL=https://grafana.jempi-qa.jembi-skunks.org +KC_JEMPI_ROOT_URL=https://jempi-web.jempi-qa.jembi-skunks.org +KC_JEMPI_SSO_ENABLED=true +KC_SUPERSET_ROOT_URL=https://superset.jempi-qa.jembi-skunks.org +KC_OPENHIM_ROOT_URL=https://openhimconsole.jempi-qa.jembi-skunks.org +GF_SERVER_DOMAIN=grafana.jempi-qa.jembi-skunks.org +KC_POSTGRES_REPLICA_SET=pgpool-1:5432,pgpool-2:5432,pgpool-3:5432 + +# Client Registry - JeMPI +JEMPI_WEB_INSTANCES=1 +REACT_APP_JEMPI_BASE_API_HOST=https://jempi-api.jempi-qa.jembi-skunks.org +REACT_APP_JEMPI_BASE_API_PORT=443 +JEMPI_SESSION_SECURE=true +JEMPI_SESSION_DOMAIN_NAME=jempi-qa.jembi-skunks.org +JEMPI_REPMGR_PARTNER_NODES=jempi-postgresql-01 +JEMPI_ASYNC_RECEIVER_INSTANCES=1 +JEMPI_SYNC_RECEIVER_INSTANCES=1 +JEMPI_PRE_PROCESSOR_INSTANCES=1 +JEMPI_CONTROLLER_INSTANCES=1 +JEMPI_EM_SCALA_INSTANCES=1 +JEMPI_LINKER_INSTANCES=3 +JEMPI_API_INSTANCES=1 +JEMPI_POSTGRES_DB=pgpool-1,pgpool-2,pgpool-3 +JEMPI_ASYNC_RECEIVER_IMAGE_TAG=dev +JEMPI_CONTROLLER_IMAGE_TAG=dev +JEMPI_LINKER_IMAGE_TAG=dev +JEMPI_API_IMAGE_TAG=dev +JEMPI_API_KC_IMAGE_TAG=dev +JEMPI_ETL_IMAGE_TAG=dev +JEMPI_WEB_VERSION=dev +JEMPI_BOOTSTRAPPER_IMAGE_TAG=dev +JEMPI_EM_SCALA_IMAGE_TAG=dev +JEMPI_POSTGRESQL_USERNAME=postgres + +# Resource limits +OPENHIM_MEMORY_LIMIT=4G +ES_MEMORY_LIMIT=20G +LOGSTASH_MEMORY_LIMIT=8G +KAFKA_MEMORY_LIMIT=8G +KAFDROP_MEMORY_LIMIT=500M + +# PG Pool +# The postgres users have to be specified so that they can be added to the pg_pool authentication interceptor +PGPOOL_POSTGRES_CUSTOM_USERS=hapi,keycloak,jempi,superset +PGPOOL_POSTGRES_CUSTOM_PASSWORDS=instant101,instant101,instant101,instant101 diff --git a/reverse-proxy-nginx/package-conf-secure/http-jempi-secure.conf b/reverse-proxy-nginx/package-conf-secure/http-jempi-secure.conf index 2b421c32..52a90e92 100644 --- a/reverse-proxy-nginx/package-conf-secure/http-jempi-secure.conf +++ b/reverse-proxy-nginx/package-conf-secure/http-jempi-secure.conf @@ -18,6 +18,8 @@ server { listen [::]:443 ssl; server_name jempi-web.*; + client_max_body_size 128M; # allows file uploads up to 128 megabytes + location /.well-known/acme-challenge/ { resolver 127.0.0.11 valid=30s; set $upstream_certbot certbot; @@ -51,6 +53,8 @@ server { listen [::]:443 ssl; server_name jempi-api.*; + client_max_body_size 128M; # allows file uploads up to 128 megabytes + location /.well-known/acme-challenge/ { resolver 127.0.0.11 valid=30s; set $upstream_certbot certbot;