Merge branch 'main' into add-automatic-cert-to-monitoring

jembi · Dec 3, 2024 · 4ed6606 · 4ed6606
2 parents 42a286b + 67f8eeb
commit 4ed6606
Show file tree

Hide file tree

Showing 11 changed files with 254 additions and 130 deletions.
diff --git a/README.md b/README.md
@@ -1,7 +1,17 @@
 # Jembi Platform
 
+OpenHIM platform is an easy way to set up, manage and operate a Health Information Exchange (HIE). Specifically, it is the following:
+
+- A toolbox of open-source tools, grouped into packages, that are used within an HIE.
+- The glue that ties these tools together. These are often in the form of OpenHIM mediators which are just microservices that talk to OpenHIM.
+- A CLI tool to deploy and manage these packages.
+
+> [!NOTE]
+> 📚 Check out the [OpenHIM platform documentation](https://jembi.gitbook.io/openhim-platform) for more information!
+
 ## Tech Used
 
+- Instant OpenHIE
 - Docker
 - Golang (cli dev)
 - Terraform (remote cluster setup)

diff --git a/config.yaml b/config.yaml
@@ -27,6 +27,7 @@ packages:
   - database-postgres
   - reprocess-mediator
   - fhir-ig-importer
+  - datalake
 
 profiles:
   - name: cdr-dw
@@ -47,6 +48,7 @@ profiles:
       - kafka-unbundler-consumer
       - fhir-ig-importer
       - reprocess-mediator
+      - datalake
     envFiles:
       - cdr-dw.env
 

diff --git a/datalake/docker-compose.cluster.yml b/datalake/docker-compose.cluster.yml
@@ -0,0 +1,82 @@
+version: "3.9"
+
+services:
+  minio-01:
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.name==${MINIO_01_PLACEMENT}"
+
+  minio-02:
+    image: ${MINIO_IMAGE}
+    entrypoint: sh
+    command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...4}/data{1...2}'
+    environment:
+      MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER}
+      MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+    hostname: minio-02
+    volumes:
+      - minio-02-data1:/data1
+      - minio-02-data2:/data2
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.name==${MINIO_02_PLACEMENT}"
+      replicas: 1
+
+  minio-03:
+    image: ${MINIO_IMAGE}
+    entrypoint: sh
+    command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...4}/data{1...2}'
+    environment:
+      MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER}
+      MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+    hostname: minio-03
+    volumes:
+      - minio-03-data1:/data1
+      - minio-03-data2:/data2
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.name==${MINIO_03_PLACEMENT}"
+      replicas: 1
+
+  minio-04:
+    image: ${MINIO_IMAGE}
+    entrypoint: sh
+    command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...4}/data{1...2}'
+    environment:
+      MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER}
+      MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+    hostname: minio-04
+    volumes:
+      - minio-04-data1:/data1
+      - minio-04-data2:/data2
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.name==${MINIO_04_PLACEMENT}"
+      replicas: 1
+
+volumes:
+  minio-02-data1:
+  minio-02-data2:
+  minio-03-data1:
+  minio-03-data2:
+  minio-04-data1:
+  minio-04-data2:
diff --git a/datalake/docker-compose.dev.yml b/datalake/docker-compose.dev.yml
@@ -0,0 +1,12 @@
+version: '3.9'
+
+services:
+  minio-01:
+    ports:
+      - target: 9001
+        published: 9005
+        mode: host
+
+      - target: 9090
+        published: 9393
+        mode: host
diff --git a/datalake/docker-compose.yml b/datalake/docker-compose.yml
@@ -0,0 +1,40 @@
+version: '3.9'
+
+services:
+  minio-01:
+    image: ${MINIO_IMAGE}
+    entrypoint: sh
+    command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" --address ":9090" http://minio-0{1..${NUM_MINIO_SERVERS}}/data{1..2}'
+    environment:
+      MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER}
+      MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
+      MINIO_BROWSER_REDIRECT_URL: ${MINIO_BROWSER_REDIRECT_URL}
+      MINIO_SERVER_URL: ${MINIO_SERVER_URL:-http://localhost:9090}
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://localhost:9090/minio/health/live" ]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+    hostname: minio-01
+    volumes:
+      - minio-01-data1:/data1
+      - minio-01-data2:/data2
+    deploy:
+      replicas: 1
+      labels:
+        - traefik.enable=true
+        - traefik.docker.network=reverse-proxy-traefik_public
+        - traefik.http.routers.minio.rule=${DOMAIN_NAME_HOST_TRAEFIK} && PathPrefix(`/minio`)
+        - traefik.http.services.minio.loadbalancer.server.port=9001
+        - traefik.http.middlewares.minio-stripprefix.stripprefix.prefixes=/minio
+        - traefik.http.routers.minio.middlewares=minio-stripprefix
+    networks:
+      public:
+networks:
+  public:
+    name: minio_public
+    external: true
+
+volumes:
+  minio-01-data1:
+  minio-01-data2:
diff --git a/datalake/package-metadata.json b/datalake/package-metadata.json
@@ -0,0 +1,22 @@
+{
+  "id": "datalake",
+  "name": "datalake",
+  "description": "Object storage for raw unstructured data, compatible with the s3 API",
+  "type": "infrastructure",
+  "version": "0.0.1",
+  "dependencies": [],
+  "environmentVariables": {
+    "NUM_MINIO_SERVERS": 1,
+    "MO_SECURITY_ADMIN_USER": "admin",
+    "MO_SECURITY_ADMIN_PASSWORD": "dev_password_only",
+    "MO_RETENTION_TIME": "15d",
+    "MINIO_BROWSER_REDIRECT_URL": "http://localhost:9001",
+    "DOMAIN_NAME_HOST_TRAEFIK": "localhost",
+    "MINIO_01_PLACEMENT": "minio-01",
+    "MINIO_02_PLACEMENT": "minio-02",
+    "MINIO_03_PLACEMENT": "minio-03",
+    "MINIO_04_PLACEMENT": "minio-04",
+    "MINIO_IMAGE": "minio/minio:RELEASE.2024-10-13T13-34-11Z.fips",
+    "MINIO_SERVER_URL": "http://localhost:9090"
+  }
+}
diff --git a/datalake/swarm.sh b/datalake/swarm.sh
@@ -0,0 +1,79 @@
+#!/bin/bash
+
+declare ACTION=""
+declare MODE=""
+declare COMPOSE_FILE_PATH=""
+declare UTILS_PATH=""
+declare STACK="datalake"
+
+function init_vars() {
+  ACTION=$1
+  MODE=$2
+
+  COMPOSE_FILE_PATH=$(
+    cd "$(dirname "${BASH_SOURCE[0]}")" || exit
+    pwd -P
+  )
+
+  UTILS_PATH="${COMPOSE_FILE_PATH}/../utils"
+
+  readonly ACTION
+  readonly MODE
+  readonly COMPOSE_FILE_PATH
+  readonly UTILS_PATH
+  readonly STACK
+}
+
+# shellcheck disable=SC1091
+function import_sources() {
+  source "${UTILS_PATH}/docker-utils.sh"
+  source "${UTILS_PATH}/log.sh"
+}
+
+function initialize_package() {
+  local package_dev_compose_filename=""
+  local minio_cluster_compose_filename=""
+  if [[ "${CLUSTERED_MODE}" == "true" ]]; then
+    minio_cluster_compose_filename="docker-compose.cluster.yml"
+    export NUM_MINIO_SERVERS=4
+  else
+    export NUM_MINIO_SERVERS=1
+  fi
+  if [[ "${MODE}" == "dev" ]]; then
+    log info "Running package in DEV mode"
+    package_dev_compose_filename="docker-compose.dev.yml"
+  else
+    log info "Running package in PROD mode"
+  fi
+
+  (
+    docker::deploy_service "$STACK" "${COMPOSE_FILE_PATH}" "docker-compose.yml" "$package_dev_compose_filename" "$minio_cluster_compose_filename"
+  ) || {
+    log error "Failed to deploy package"
+    exit 1
+  }
+}
+
+function destroy_package() {
+  docker::stack_destroy "$STACK"
+}
+
+main() {
+  init_vars "$@"
+  import_sources
+
+  if [[ "${ACTION}" == "init" ]] || [[ "${ACTION}" == "up" ]]; then
+    log info "Running package in Single node mode"
+    initialize_package
+  elif [[ "${ACTION}" == "down" ]]; then
+    log info "Scaling down package"
+    docker::scale_services "$STACK" 0
+  elif [[ "${ACTION}" == "destroy" ]]; then
+    log info "Destroying package"
+    destroy_package
+  else
+    log error "Valid options are: init, up, down, or destroy"
+  fi
+}
+
+main "$@"
diff --git a/monitoring/docker-compose.cluster.yml b/monitoring/docker-compose.cluster.yml
@@ -32,83 +32,6 @@ services:
       public:
       default:
 
-  minio-01:
-    deploy:
-      placement:
-        constraints:
-          - "node.labels.name==${MINIO_01_PLACEMENT}"
-
-  minio-02:
-    image: ${MINIO_IMAGE}
-    entrypoint: sh
-    command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...4}/data{1...2}'
-    environment:
-      MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER}
-      MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
-      interval: 30s
-      timeout: 20s
-      retries: 3
-    hostname: minio-02
-    volumes:
-      - minio-02-data1:/data1
-      - minio-02-data2:/data2
-    deploy:
-      placement:
-        constraints:
-          - "node.labels.name==${MINIO_02_PLACEMENT}"
-      replicas: 1
-
-  minio-03:
-    image: ${MINIO_IMAGE}
-    entrypoint: sh
-    command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...4}/data{1...2}'
-    environment:
-      MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER}
-      MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
-      interval: 30s
-      timeout: 20s
-      retries: 3
-    hostname: minio-03
-    volumes:
-      - minio-03-data1:/data1
-      - minio-03-data2:/data2
-    deploy:
-      placement:
-        constraints:
-          - "node.labels.name==${MINIO_03_PLACEMENT}"
-      replicas: 1
-
-  minio-04:
-    image: ${MINIO_IMAGE}
-    entrypoint: sh
-    command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...4}/data{1...2}'
-    environment:
-      MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER}
-      MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
-      interval: 30s
-      timeout: 20s
-      retries: 3
-    hostname: minio-04
-    volumes:
-      - minio-04-data1:/data1
-      - minio-04-data2:/data2
-    deploy:
-      placement:
-        constraints:
-          - "node.labels.name==${MINIO_04_PLACEMENT}"
-      replicas: 1
 
 volumes:
   prometheus_data_backup:
-  minio-02-data1:
-  minio-02-data2:
-  minio-03-data1:
-  minio-03-data2:
-  minio-04-data1:
-  minio-04-data2:
diff --git a/monitoring/docker-compose.dev.yml b/monitoring/docker-compose.dev.yml
@@ -18,9 +18,3 @@ services:
       - target: 3100
         published: 3100
         mode: host
-
-  minio-01:
-    ports:
-      - target: 9001
-        published: 9005
-        mode: host