diff --git a/README.md b/README.md index 1a2272ca..e404eb8d 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,17 @@ # Jembi Platform +OpenHIM platform is an easy way to set up, manage and operate a Health Information Exchange (HIE). Specifically, it is the following: + +- A toolbox of open-source tools, grouped into packages, that are used within an HIE. +- The glue that ties these tools together. These are often in the form of OpenHIM mediators which are just microservices that talk to OpenHIM. +- A CLI tool to deploy and manage these packages. + +> [!NOTE] +> 📚 Check out the [OpenHIM platform documentation](https://jembi.gitbook.io/openhim-platform) for more information! + ## Tech Used +- Instant OpenHIE - Docker - Golang (cli dev) - Terraform (remote cluster setup) diff --git a/config.yaml b/config.yaml index 3d77445b..8c6372b3 100644 --- a/config.yaml +++ b/config.yaml @@ -27,6 +27,7 @@ packages: - database-postgres - reprocess-mediator - fhir-ig-importer + - datalake profiles: - name: cdr-dw @@ -47,6 +48,7 @@ profiles: - kafka-unbundler-consumer - fhir-ig-importer - reprocess-mediator + - datalake envFiles: - cdr-dw.env diff --git a/datalake/docker-compose.cluster.yml b/datalake/docker-compose.cluster.yml new file mode 100644 index 00000000..da2833e9 --- /dev/null +++ b/datalake/docker-compose.cluster.yml @@ -0,0 +1,82 @@ +version: "3.9" + +services: + minio-01: + deploy: + placement: + constraints: + - "node.labels.name==${MINIO_01_PLACEMENT}" + + minio-02: + image: ${MINIO_IMAGE} + entrypoint: sh + command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...4}/data{1...2}' + environment: + MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER} + MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD} + healthcheck: + test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ] + interval: 30s + timeout: 20s + retries: 3 + hostname: minio-02 + volumes: + - minio-02-data1:/data1 + - minio-02-data2:/data2 + deploy: + placement: + constraints: + - "node.labels.name==${MINIO_02_PLACEMENT}" + replicas: 1 + + minio-03: + image: ${MINIO_IMAGE} + entrypoint: sh + command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...4}/data{1...2}' + environment: + MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER} + MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD} + healthcheck: + test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ] + interval: 30s + timeout: 20s + retries: 3 + hostname: minio-03 + volumes: + - minio-03-data1:/data1 + - minio-03-data2:/data2 + deploy: + placement: + constraints: + - "node.labels.name==${MINIO_03_PLACEMENT}" + replicas: 1 + + minio-04: + image: ${MINIO_IMAGE} + entrypoint: sh + command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...4}/data{1...2}' + environment: + MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER} + MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD} + healthcheck: + test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ] + interval: 30s + timeout: 20s + retries: 3 + hostname: minio-04 + volumes: + - minio-04-data1:/data1 + - minio-04-data2:/data2 + deploy: + placement: + constraints: + - "node.labels.name==${MINIO_04_PLACEMENT}" + replicas: 1 + +volumes: + minio-02-data1: + minio-02-data2: + minio-03-data1: + minio-03-data2: + minio-04-data1: + minio-04-data2: diff --git a/datalake/docker-compose.dev.yml b/datalake/docker-compose.dev.yml new file mode 100644 index 00000000..f5999410 --- /dev/null +++ b/datalake/docker-compose.dev.yml @@ -0,0 +1,12 @@ +version: '3.9' + +services: + minio-01: + ports: + - target: 9001 + published: 9005 + mode: host + + - target: 9090 + published: 9393 + mode: host diff --git a/datalake/docker-compose.yml b/datalake/docker-compose.yml new file mode 100644 index 00000000..0c1f2cb2 --- /dev/null +++ b/datalake/docker-compose.yml @@ -0,0 +1,40 @@ +version: '3.9' + +services: + minio-01: + image: ${MINIO_IMAGE} + entrypoint: sh + command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" --address ":9090" http://minio-0{1..${NUM_MINIO_SERVERS}}/data{1..2}' + environment: + MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER} + MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD} + MINIO_BROWSER_REDIRECT_URL: ${MINIO_BROWSER_REDIRECT_URL} + MINIO_SERVER_URL: ${MINIO_SERVER_URL:-http://localhost:9090} + healthcheck: + test: [ "CMD", "curl", "-f", "http://localhost:9090/minio/health/live" ] + interval: 30s + timeout: 20s + retries: 3 + hostname: minio-01 + volumes: + - minio-01-data1:/data1 + - minio-01-data2:/data2 + deploy: + replicas: 1 + labels: + - traefik.enable=true + - traefik.docker.network=reverse-proxy-traefik_public + - traefik.http.routers.minio.rule=${DOMAIN_NAME_HOST_TRAEFIK} && PathPrefix(`/minio`) + - traefik.http.services.minio.loadbalancer.server.port=9001 + - traefik.http.middlewares.minio-stripprefix.stripprefix.prefixes=/minio + - traefik.http.routers.minio.middlewares=minio-stripprefix + networks: + public: +networks: + public: + name: minio_public + external: true + +volumes: + minio-01-data1: + minio-01-data2: diff --git a/datalake/package-metadata.json b/datalake/package-metadata.json new file mode 100644 index 00000000..0c51dd13 --- /dev/null +++ b/datalake/package-metadata.json @@ -0,0 +1,22 @@ +{ + "id": "datalake", + "name": "datalake", + "description": "Object storage for raw unstructured data, compatible with the s3 API", + "type": "infrastructure", + "version": "0.0.1", + "dependencies": [], + "environmentVariables": { + "NUM_MINIO_SERVERS": 1, + "MO_SECURITY_ADMIN_USER": "admin", + "MO_SECURITY_ADMIN_PASSWORD": "dev_password_only", + "MO_RETENTION_TIME": "15d", + "MINIO_BROWSER_REDIRECT_URL": "http://localhost:9001", + "DOMAIN_NAME_HOST_TRAEFIK": "localhost", + "MINIO_01_PLACEMENT": "minio-01", + "MINIO_02_PLACEMENT": "minio-02", + "MINIO_03_PLACEMENT": "minio-03", + "MINIO_04_PLACEMENT": "minio-04", + "MINIO_IMAGE": "minio/minio:RELEASE.2024-10-13T13-34-11Z.fips", + "MINIO_SERVER_URL": "http://localhost:9090" + } +} diff --git a/datalake/swarm.sh b/datalake/swarm.sh new file mode 100644 index 00000000..cdee2333 --- /dev/null +++ b/datalake/swarm.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +declare ACTION="" +declare MODE="" +declare COMPOSE_FILE_PATH="" +declare UTILS_PATH="" +declare STACK="datalake" + +function init_vars() { + ACTION=$1 + MODE=$2 + + COMPOSE_FILE_PATH=$( + cd "$(dirname "${BASH_SOURCE[0]}")" || exit + pwd -P + ) + + UTILS_PATH="${COMPOSE_FILE_PATH}/../utils" + + readonly ACTION + readonly MODE + readonly COMPOSE_FILE_PATH + readonly UTILS_PATH + readonly STACK +} + +# shellcheck disable=SC1091 +function import_sources() { + source "${UTILS_PATH}/docker-utils.sh" + source "${UTILS_PATH}/log.sh" +} + +function initialize_package() { + local package_dev_compose_filename="" + local minio_cluster_compose_filename="" + if [[ "${CLUSTERED_MODE}" == "true" ]]; then + minio_cluster_compose_filename="docker-compose.cluster.yml" + export NUM_MINIO_SERVERS=4 + else + export NUM_MINIO_SERVERS=1 + fi + if [[ "${MODE}" == "dev" ]]; then + log info "Running package in DEV mode" + package_dev_compose_filename="docker-compose.dev.yml" + else + log info "Running package in PROD mode" + fi + + ( + docker::deploy_service "$STACK" "${COMPOSE_FILE_PATH}" "docker-compose.yml" "$package_dev_compose_filename" "$minio_cluster_compose_filename" + ) || { + log error "Failed to deploy package" + exit 1 + } +} + +function destroy_package() { + docker::stack_destroy "$STACK" +} + +main() { + init_vars "$@" + import_sources + + if [[ "${ACTION}" == "init" ]] || [[ "${ACTION}" == "up" ]]; then + log info "Running package in Single node mode" + initialize_package + elif [[ "${ACTION}" == "down" ]]; then + log info "Scaling down package" + docker::scale_services "$STACK" 0 + elif [[ "${ACTION}" == "destroy" ]]; then + log info "Destroying package" + destroy_package + else + log error "Valid options are: init, up, down, or destroy" + fi +} + +main "$@" diff --git a/monitoring/docker-compose.cluster.yml b/monitoring/docker-compose.cluster.yml index bbbb1996..44001e73 100644 --- a/monitoring/docker-compose.cluster.yml +++ b/monitoring/docker-compose.cluster.yml @@ -32,83 +32,6 @@ services: public: default: - minio-01: - deploy: - placement: - constraints: - - "node.labels.name==${MINIO_01_PLACEMENT}" - - minio-02: - image: ${MINIO_IMAGE} - entrypoint: sh - command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...4}/data{1...2}' - environment: - MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER} - MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD} - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] - interval: 30s - timeout: 20s - retries: 3 - hostname: minio-02 - volumes: - - minio-02-data1:/data1 - - minio-02-data2:/data2 - deploy: - placement: - constraints: - - "node.labels.name==${MINIO_02_PLACEMENT}" - replicas: 1 - - minio-03: - image: ${MINIO_IMAGE} - entrypoint: sh - command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...4}/data{1...2}' - environment: - MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER} - MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD} - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] - interval: 30s - timeout: 20s - retries: 3 - hostname: minio-03 - volumes: - - minio-03-data1:/data1 - - minio-03-data2:/data2 - deploy: - placement: - constraints: - - "node.labels.name==${MINIO_03_PLACEMENT}" - replicas: 1 - - minio-04: - image: ${MINIO_IMAGE} - entrypoint: sh - command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...4}/data{1...2}' - environment: - MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER} - MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD} - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] - interval: 30s - timeout: 20s - retries: 3 - hostname: minio-04 - volumes: - - minio-04-data1:/data1 - - minio-04-data2:/data2 - deploy: - placement: - constraints: - - "node.labels.name==${MINIO_04_PLACEMENT}" - replicas: 1 volumes: prometheus_data_backup: - minio-02-data1: - minio-02-data2: - minio-03-data1: - minio-03-data2: - minio-04-data1: - minio-04-data2: diff --git a/monitoring/docker-compose.dev.yml b/monitoring/docker-compose.dev.yml index 38b4c708..89fa58bb 100644 --- a/monitoring/docker-compose.dev.yml +++ b/monitoring/docker-compose.dev.yml @@ -18,9 +18,3 @@ services: - target: 3100 published: 3100 mode: host - - minio-01: - ports: - - target: 9001 - published: 9005 - mode: host diff --git a/monitoring/docker-compose.yml b/monitoring/docker-compose.yml index 9871a70e..20f2f89f 100644 --- a/monitoring/docker-compose.yml +++ b/monitoring/docker-compose.yml @@ -136,6 +136,8 @@ services: - target: /etc/loki/loki-config.yml source: loki-config.yml command: -config.file=/etc/loki/loki-config.yml -config.expand-env=true + networks: + minio: deploy: labels: - prometheus-job-service=loki @@ -152,45 +154,6 @@ services: command: -config.file=/etc/promtail/promtail-config.yml deploy: mode: global - - minio-01: - image: quay.io/minio/minio:RELEASE.2022-10-24T18-35-07Z - entrypoint: sh - command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...${NUM_MINIO_SERVERS}}/data{1...2}' - environment: - MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER} - MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD} - MINIO_BROWSER_REDIRECT_URL: ${MINIO_BROWSER_REDIRECT_URL} - MINIO_SERVER_URL: http://localhost:9000 - healthcheck: - test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ] - interval: 30s - timeout: 20s - retries: 3 - hostname: minio-01 - volumes: - - minio-01-data1:/data1 - - minio-01-data2:/data2 - deploy: - replicas: 1 - labels: - - traefik.enable=true - - traefik.docker.network=reverse-proxy-traefik_public - - traefik.http.routers.minio.service=minio - - traefik.http.routers.minio.rule=Host(`${DOMAIN_NAME_HOST_TRAEFIK}`) && PathPrefix(`/minio`) - - traefik.http.services.minio.loadbalancer.server.port=9001 - - traefik.http.routers.minio.tls=${TLS} - - traefik.http.services.minio.loadbalancer.server.scheme=http - - traefik.http.routers.minio.entrypoints=${WEB_ENTRY_POINT} - - traefik.http.routers.minio.tls.certresolver=le - - traefik.http.middlewares.minio-stripprefix.stripprefix.prefixes=/minio - - traefik.http.routers.minio.middlewares=minio-stripprefix - networks: - reverse-proxy: - traefik: - default: - - configs: grafana.ini: file: ./grafana/grafana.ini @@ -272,8 +235,7 @@ volumes: prometheus-data: grafana-data: loki-data: - minio-01-data1: - minio-01-data2: + networks: @@ -289,4 +251,7 @@ networks: traefik: name: reverse-proxy-traefik_public external: true + minio: + name: minio_public + external: true default: diff --git a/monitoring/package-metadata.json b/monitoring/package-metadata.json index 82dc6b54..31691759 100644 --- a/monitoring/package-metadata.json +++ b/monitoring/package-metadata.json @@ -4,19 +4,14 @@ "description": "A package for monitoring the platform services", "type": "infrastructure", "version": "0.0.1", - "dependencies": [], + "dependencies": ["datalake"], "environmentVariables": { "GRAFANA_IMAGE": "grafana/grafana-oss:9.2.3", "LOKI_IMAGE": "grafana/loki:2.6.1", "PROMTAIL_IMAGE": "grafana/promtail:2.6.1", "PROMETHEUS_BACKUP_IMAGE": "prom/prometheus:v2.38.0", - "MINIO_IMAGE": "quay.io/minio/minio:RELEASE.2022-10-24T18-35-07Z", "PROMETHEUS_PLACEMENT": "node-1", "PROMETHEUS_BACKUP_PLACEMENT": "node-1", - "MINIO_01_PLACEMENT": "node-1", - "MINIO_02_PLACEMENT": "node-1", - "MINIO_03_PLACEMENT": "node-2", - "MINIO_04_PLACEMENT": "node-3", "GF_SECURITY_ADMIN_USER": "admin", "GF_SECURITY_ADMIN_PASSWORD": "dev_password_only", "GF_SMTP_ENABLED": "false",