diff --git a/climate-mediator/docker-compose.dev.yml b/climate-mediator/docker-compose.dev.yml new file mode 100644 index 00000000..823b1f23 --- /dev/null +++ b/climate-mediator/docker-compose.dev.yml @@ -0,0 +1,8 @@ +version: '3.9' + +services: + climate-mediator: + ports: + - target: 3000 + published: 3011 + mode: host diff --git a/climate-mediator/docker-compose.yml b/climate-mediator/docker-compose.yml new file mode 100644 index 00000000..fc10a2df --- /dev/null +++ b/climate-mediator/docker-compose.yml @@ -0,0 +1,37 @@ +version: "3.9" + +services: + climate-mediator: + image: ${CLIMATE_MEDIATOR_IMAGE} + networks: + openhim: + clickhouse: + minio: + public: + default: + environment: + OPENHIM_USERNAME: ${OPENHIM_USERNAME} + OPENHIM_PASSWORD: ${OPENHIM_PASSWORD} + REGISTER_MEDIATOR: ${REGISTER_MEDIATOR} + OPENHIM_MEDIATOR_URL: ${OPENHIM_MEDIATOR_URL} + TRUST_SELF_SIGNED: ${TRUST_SELF_SIGNED} + MINIO_ENDPOINT: ${MINIO_ENDPOINT} + MINIO_PORT: ${MINIO_PORT} + MINIO_BUCKET: ${MINIO_BUCKET} + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + +networks: + openhim: + name: openhim_public + external: true + clickhouse: + name: clickhouse_public + external: true + minio: + name: minio_public + external: true + public: + name: climate_public + external: true + default: diff --git a/climate-mediator/importer/docker-compose.config.yml b/climate-mediator/importer/docker-compose.config.yml new file mode 100644 index 00000000..3a62f5cc --- /dev/null +++ b/climate-mediator/importer/docker-compose.config.yml @@ -0,0 +1,36 @@ +version: "3.9" + +services: + climate-mediator-config-importer: + image: ${CLIMATE_MEDIATOR_CONFIG_IMPORTER_IMAGE} + networks: + openhim: + default: + environment: + OPENHIM_API_USERNAME: ${OPENHIM_USERNAME} + OPENHIM_API_PASSWORD: ${OPENHIM_PASSWORD} + NODE_TLS_REJECT_UNAUTHORIZED: 0 + command: sh -c "node openhimConfig.js" + configs: + - source: climate-mediator-config-importer-openhimConfig.js + target: /openhimConfig.js + - source: climate-mediator-config-importer-openhim-import.json + target: /openhim-import.json + +configs: + climate-mediator-config-importer-openhimConfig.js: + file: ./volume/openhimConfig.js + name: climate-mediator-config-importer-openhimConfig.js-${climate_mediator_config_importer_openhimConfig_js_DIGEST:?err} + labels: + name: climate-mediator + climate-mediator-config-importer-openhim-import.json: + file: ./volume/openhim-import.json + name: climate-mediator-config-importer-openhim-import.json-${climate_mediator_config_importer_openhim_import_js_DIGEST:?err} + labels: + name: climate-mediator + +networks: + openhim: + name: openhim_public + external: true + default: diff --git a/climate-mediator/importer/volume/openhim-import.json b/climate-mediator/importer/volume/openhim-import.json new file mode 100644 index 00000000..d9ed3ccb --- /dev/null +++ b/climate-mediator/importer/volume/openhim-import.json @@ -0,0 +1,92 @@ +{ + "Users": [ + { + "firstname": "Super", + "surname": "User", + "email": "root@openhim.org", + "provider": "token", + "groups": ["admin"], + "passwordAlgorithm": "sha512", + "passwordHash": "ea3824f17cf1379eb118a36bc7c8cf0f45712e2af7748567fca5313dec6fa66d61064e82a5e5cb88e998486ee3c7d0dac235bbeda8c341d6edc1c77406be2ab6", + "passwordSalt": "d4f622c0404f09bd959bfb263efa3452", + "expiry": null, + "locked": false, + "token": null, + "tokenType": null + } + ], + "Clients": [ + { + "clientID": "instant-client", + "name": "Instant Client", + "roles": ["instant"], + "customTokenID": "test" + } + ], + "Channels": [ + { + "name": "Climate Mediator", + "urlPattern": "^/climate.*$", + "routes": [ + { + "name": "Climate Mediator Endpoint", + "host": "climate-mediator", + "port": "3000", + "primary": true, + "type": "http", + "path": "/climate/upload" + } + ], + "allow": ["instant"], + "methods": ["POST", "GET"], + "type": "http", + "updatedBy": { + "id": "65d47e98223a0672c3c9808b", + "name": "Super User" + } + } + ], + "Mediators": [ + { + "urn": "urn:mediator:climate-mediator", + "version": "0.0.1", + "name": "Climate mediator", + "description": "A mediator handling interactions between the OpenHIM Core service and climate related data", + "endpoints": [ + { + "name": "Climate Endpoint", + "type": "http", + "status": "enabled", + "host": "climate-mediator", + "port": 3000, + "path": "/climate", + "primary": true, + "forwardAuthHeader": false + } + ], + "defaultChannelConfig": [ + { + "name": "Climate Mediator", + "urlPattern": "^/climate.*$", + "routes": [ + { + "name": "Climate Mediator Endpoint", + "host": "climate-mediator", + "port": "3000", + "primary": true, + "type": "http", + "path": "/climate/upload" + } + ], + "allow": ["instant"], + "methods": ["POST", "GET"], + "type": "http" + } + ], + "configDefs": [], + "_lastHeartbeat": "2024-12-14T12:23:22.769Z", + "_uptime": 5650.734193405 + } + ], + "ContactGroups": [] +} diff --git a/climate-mediator/importer/volume/openhimConfig.js b/climate-mediator/importer/volume/openhimConfig.js new file mode 100644 index 00000000..b3891c21 --- /dev/null +++ b/climate-mediator/importer/volume/openhimConfig.js @@ -0,0 +1,54 @@ +'use strict' + +const fs = require('fs') +const https = require('https') +const path = require('path') + +const OPENHIM_CORE_SERVICE_NAME = 'openhim-core' +const OPENHIM_MEDIATOR_API_PORT = 8080 +const OPENHIM_API_PASSWORD = + process.env.OPENHIM_API_PASSWORD || 'instant101' +const OPENHIM_API_USERNAME = + process.env.OPENHIM_API_USERNAME || 'root@openhim.org' + +const authHeader = new Buffer.from( + `${OPENHIM_API_USERNAME}:${OPENHIM_API_PASSWORD}` +).toString('base64') + +const jsonData = JSON.parse( + fs.readFileSync(path.resolve(__dirname, 'openhim-import.json')) +) + +const data = JSON.stringify(jsonData) + +const options = { + protocol: 'https:', + hostname: OPENHIM_CORE_SERVICE_NAME, + port: OPENHIM_MEDIATOR_API_PORT, + path: '/metadata', + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Content-Length': data.length, + Authorization: `Basic ${authHeader}` + } +} + +const req = https.request(options, res => { + if (res.statusCode == 401) { + throw new Error(`Incorrect OpenHIM API credentials`) + } + + if (res.statusCode != 201) { + throw new Error(`Failed to import OpenHIM config: ${res.statusCode}`) + } + + console.log('Successfully Imported OpenHIM Config') +}) + +req.on('error', error => { + throw new Error(`Failed to import OpenHIM config: ${error}`) +}) + +req.write(data) +req.end() diff --git a/climate-mediator/package-metadata.json b/climate-mediator/package-metadata.json new file mode 100644 index 00000000..cb1dbedd --- /dev/null +++ b/climate-mediator/package-metadata.json @@ -0,0 +1,22 @@ +{ + "id": "climate-mediator", + "name": "Climate Mediator", + "description": "This package creates a mediator service that is used in processing climate related data", + "type": "infrastructure", + "version": "0.0.1", + "dependencies": ["interoperability-layer-openhim", "datalake"], + "environmentVariables": { + "TRUST_SELF_SIGNED": "true", + "OPENHIM_MEDIATOR_URL": "https://openhim-core:8080", + "OPENHIM_USERNAME": "root@openhim.org", + "OPENHIM_PASSWORD": "instant101", + "REGISTER_MEDIATOR": "true", + "CLIMATE_MEDIATOR_IMAGE": "jembi/climate-mediator:latest", + "CLIMATE_MEDIATOR_CONFIG_IMPORTER_IMAGE": "node:erbium-alpine", + "MINIO_BUCKET": "climate-mediator", + "MINIO_ACCESS_KEY": "tCroZpZ3usDUcvPM3QT6", + "MINIO_SECRET_KEY": "suVjMHUpVIGyWx8fFJHTiZiT88dHhKgVpzvYTOKK", + "MINIO_ENDPOINT": "minio-01", + "MINIO_PORT": "9090" + } +} diff --git a/climate-mediator/swarm.sh b/climate-mediator/swarm.sh new file mode 100644 index 00000000..c5387024 --- /dev/null +++ b/climate-mediator/swarm.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +declare ACTION="" +declare MODE="" +declare COMPOSE_FILE_PATH="" +declare UTILS_PATH="" +declare STACK="climate" + +function init_vars() { + ACTION=$1 + MODE=$2 + + COMPOSE_FILE_PATH=$( + cd "$(dirname "${BASH_SOURCE[0]}")" || exit + pwd -P + ) + + UTILS_PATH="${COMPOSE_FILE_PATH}/../utils" + + readonly ACTION + readonly MODE + readonly COMPOSE_FILE_PATH + readonly UTILS_PATH + readonly STACK +} + +# shellcheck disable=SC1091 +function import_sources() { + source "${UTILS_PATH}/docker-utils.sh" + source "${UTILS_PATH}/log.sh" +} + +function initialize_package() { + local climate_mediator_dev_compose_filename="" + + if [[ "${MODE}" == "dev" ]]; then + log info "Running package in DEV mode" + climate_mediator_dev_compose_filename="docker-compose.dev.yml" + else + log info "Running package in PROD mode" + fi + + ( + docker::deploy_service "$STACK" "${COMPOSE_FILE_PATH}" "docker-compose.yml" "$climate_mediator_dev_compose_filename" + ) || { + log error "Failed to deploy package" + exit 1 + } + docker::deploy_config_importer $STACK "$COMPOSE_FILE_PATH/importer/docker-compose.config.yml" "climate-mediator-config-importer" "climate-mediator" +} + +function destroy_package() { + docker::stack_destroy "$STACK" + + docker::prune_configs "climate-mediator" +} + +main() { + init_vars "$@" + import_sources + + if [[ "${ACTION}" == "init" ]] || [[ "${ACTION}" == "up" ]]; then + if [[ "${CLUSTERED_MODE}" == "true" ]]; then + log info "Running package in Cluster node mode" + else + log info "Running package in Single node mode" + fi + + initialize_package + elif [[ "${ACTION}" == "down" ]]; then + log info "Scaling down package" + + docker::scale_services "$STACK" 0 + elif [[ "${ACTION}" == "destroy" ]]; then + log info "Destroying package" + destroy_package + else + log error "Valid options are: init, up, down, or destroy" + fi +} + +main "$@" diff --git a/config.yaml b/config.yaml index 3d77445b..01836fa3 100644 --- a/config.yaml +++ b/config.yaml @@ -27,6 +27,8 @@ packages: - database-postgres - reprocess-mediator - fhir-ig-importer + - datalake + - climate-mediator profiles: - name: cdr-dw @@ -47,6 +49,7 @@ profiles: - kafka-unbundler-consumer - fhir-ig-importer - reprocess-mediator + - datalake envFiles: - cdr-dw.env diff --git a/datalake/docker-compose.cluster.yml b/datalake/docker-compose.cluster.yml new file mode 100644 index 00000000..da2833e9 --- /dev/null +++ b/datalake/docker-compose.cluster.yml @@ -0,0 +1,82 @@ +version: "3.9" + +services: + minio-01: + deploy: + placement: + constraints: + - "node.labels.name==${MINIO_01_PLACEMENT}" + + minio-02: + image: ${MINIO_IMAGE} + entrypoint: sh + command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...4}/data{1...2}' + environment: + MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER} + MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD} + healthcheck: + test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ] + interval: 30s + timeout: 20s + retries: 3 + hostname: minio-02 + volumes: + - minio-02-data1:/data1 + - minio-02-data2:/data2 + deploy: + placement: + constraints: + - "node.labels.name==${MINIO_02_PLACEMENT}" + replicas: 1 + + minio-03: + image: ${MINIO_IMAGE} + entrypoint: sh + command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...4}/data{1...2}' + environment: + MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER} + MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD} + healthcheck: + test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ] + interval: 30s + timeout: 20s + retries: 3 + hostname: minio-03 + volumes: + - minio-03-data1:/data1 + - minio-03-data2:/data2 + deploy: + placement: + constraints: + - "node.labels.name==${MINIO_03_PLACEMENT}" + replicas: 1 + + minio-04: + image: ${MINIO_IMAGE} + entrypoint: sh + command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...4}/data{1...2}' + environment: + MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER} + MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD} + healthcheck: + test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ] + interval: 30s + timeout: 20s + retries: 3 + hostname: minio-04 + volumes: + - minio-04-data1:/data1 + - minio-04-data2:/data2 + deploy: + placement: + constraints: + - "node.labels.name==${MINIO_04_PLACEMENT}" + replicas: 1 + +volumes: + minio-02-data1: + minio-02-data2: + minio-03-data1: + minio-03-data2: + minio-04-data1: + minio-04-data2: diff --git a/datalake/docker-compose.dev.yml b/datalake/docker-compose.dev.yml new file mode 100644 index 00000000..c4e3b38e --- /dev/null +++ b/datalake/docker-compose.dev.yml @@ -0,0 +1,12 @@ +version: '3.9' + +services: + minio-01: + ports: + - target: 9001 + published: 9005 + mode: host + + - target: 9090 + published: 9090 + mode: host diff --git a/datalake/docker-compose.yml b/datalake/docker-compose.yml new file mode 100644 index 00000000..97b2c962 --- /dev/null +++ b/datalake/docker-compose.yml @@ -0,0 +1,40 @@ +version: '3.9' + +services: + minio-01: + image: ${MINIO_IMAGE} + entrypoint: sh + command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" --address ":9090" http://minio-0{1...${NUM_MINIO_SERVERS}}/data{1...2}' + environment: + MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER} + MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD} + MINIO_BROWSER_REDIRECT_URL: ${MINIO_BROWSER_REDIRECT_URL} + MINIO_SERVER_URL: ${MINIO_SERVER_URL:-http://localhost:9000} + healthcheck: + test: [ "CMD", "curl", "-f", "http://localhost:9090/minio/health/live" ] + interval: 30s + timeout: 20s + retries: 3 + hostname: minio-01 + volumes: + - minio-01-data1:/data1 + - minio-01-data2:/data2 + deploy: + replicas: 1 + labels: + - traefik.enable=true + - traefik.docker.network=reverse-proxy-traefik_public + - traefik.http.routers.minio.rule=${DOMAIN_NAME_HOST_TRAEFIK} && PathPrefix(`/minio`) + - traefik.http.services.minio.loadbalancer.server.port=9001 + - traefik.http.middlewares.minio-stripprefix.stripprefix.prefixes=/minio + - traefik.http.routers.minio.middlewares=minio-stripprefix + networks: + public: +networks: + public: + name: minio_public + external: true + +volumes: + minio-01-data1: + minio-01-data2: diff --git a/datalake/package-metadata.json b/datalake/package-metadata.json new file mode 100644 index 00000000..fd3a2981 --- /dev/null +++ b/datalake/package-metadata.json @@ -0,0 +1,22 @@ +{ + "id": "datalake", + "name": "datalake", + "description": "Object storage for raw unstructured data, compatible with the s3 API", + "type": "infrastructure", + "version": "0.0.1", + "dependencies": [], + "environmentVariables": { + "NUM_MINIO_SERVERS": 1, + "MO_SECURITY_ADMIN_USER": "admin", + "MO_SECURITY_ADMIN_PASSWORD": "dev_password_only", + "MO_RETENTION_TIME": "15d", + "MINIO_BROWSER_REDIRECT_URL": "http://localhost:9001", + "DOMAIN_NAME_HOST_TRAEFIK": "localhost", + "MINIO_01_PLACEMENT": "minio-01", + "MINIO_02_PLACEMENT": "minio-02", + "MINIO_03_PLACEMENT": "minio-03", + "MINIO_04_PLACEMENT": "minio-04", + "MINIO_IMAGE": "minio/minio:RELEASE.2024-10-13T13-34-11Z.fips", + "MINIO_SERVER_URL": "http://localhost:9000" + } +} diff --git a/datalake/swarm.sh b/datalake/swarm.sh new file mode 100644 index 00000000..cdee2333 --- /dev/null +++ b/datalake/swarm.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +declare ACTION="" +declare MODE="" +declare COMPOSE_FILE_PATH="" +declare UTILS_PATH="" +declare STACK="datalake" + +function init_vars() { + ACTION=$1 + MODE=$2 + + COMPOSE_FILE_PATH=$( + cd "$(dirname "${BASH_SOURCE[0]}")" || exit + pwd -P + ) + + UTILS_PATH="${COMPOSE_FILE_PATH}/../utils" + + readonly ACTION + readonly MODE + readonly COMPOSE_FILE_PATH + readonly UTILS_PATH + readonly STACK +} + +# shellcheck disable=SC1091 +function import_sources() { + source "${UTILS_PATH}/docker-utils.sh" + source "${UTILS_PATH}/log.sh" +} + +function initialize_package() { + local package_dev_compose_filename="" + local minio_cluster_compose_filename="" + if [[ "${CLUSTERED_MODE}" == "true" ]]; then + minio_cluster_compose_filename="docker-compose.cluster.yml" + export NUM_MINIO_SERVERS=4 + else + export NUM_MINIO_SERVERS=1 + fi + if [[ "${MODE}" == "dev" ]]; then + log info "Running package in DEV mode" + package_dev_compose_filename="docker-compose.dev.yml" + else + log info "Running package in PROD mode" + fi + + ( + docker::deploy_service "$STACK" "${COMPOSE_FILE_PATH}" "docker-compose.yml" "$package_dev_compose_filename" "$minio_cluster_compose_filename" + ) || { + log error "Failed to deploy package" + exit 1 + } +} + +function destroy_package() { + docker::stack_destroy "$STACK" +} + +main() { + init_vars "$@" + import_sources + + if [[ "${ACTION}" == "init" ]] || [[ "${ACTION}" == "up" ]]; then + log info "Running package in Single node mode" + initialize_package + elif [[ "${ACTION}" == "down" ]]; then + log info "Scaling down package" + docker::scale_services "$STACK" 0 + elif [[ "${ACTION}" == "destroy" ]]; then + log info "Destroying package" + destroy_package + else + log error "Valid options are: init, up, down, or destroy" + fi +} + +main "$@" diff --git a/monitoring/docker-compose.cluster.yml b/monitoring/docker-compose.cluster.yml index bbbb1996..44001e73 100644 --- a/monitoring/docker-compose.cluster.yml +++ b/monitoring/docker-compose.cluster.yml @@ -32,83 +32,6 @@ services: public: default: - minio-01: - deploy: - placement: - constraints: - - "node.labels.name==${MINIO_01_PLACEMENT}" - - minio-02: - image: ${MINIO_IMAGE} - entrypoint: sh - command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...4}/data{1...2}' - environment: - MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER} - MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD} - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] - interval: 30s - timeout: 20s - retries: 3 - hostname: minio-02 - volumes: - - minio-02-data1:/data1 - - minio-02-data2:/data2 - deploy: - placement: - constraints: - - "node.labels.name==${MINIO_02_PLACEMENT}" - replicas: 1 - - minio-03: - image: ${MINIO_IMAGE} - entrypoint: sh - command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...4}/data{1...2}' - environment: - MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER} - MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD} - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] - interval: 30s - timeout: 20s - retries: 3 - hostname: minio-03 - volumes: - - minio-03-data1:/data1 - - minio-03-data2:/data2 - deploy: - placement: - constraints: - - "node.labels.name==${MINIO_03_PLACEMENT}" - replicas: 1 - - minio-04: - image: ${MINIO_IMAGE} - entrypoint: sh - command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...4}/data{1...2}' - environment: - MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER} - MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD} - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] - interval: 30s - timeout: 20s - retries: 3 - hostname: minio-04 - volumes: - - minio-04-data1:/data1 - - minio-04-data2:/data2 - deploy: - placement: - constraints: - - "node.labels.name==${MINIO_04_PLACEMENT}" - replicas: 1 volumes: prometheus_data_backup: - minio-02-data1: - minio-02-data2: - minio-03-data1: - minio-03-data2: - minio-04-data1: - minio-04-data2: diff --git a/monitoring/docker-compose.dev.yml b/monitoring/docker-compose.dev.yml index 38b4c708..89fa58bb 100644 --- a/monitoring/docker-compose.dev.yml +++ b/monitoring/docker-compose.dev.yml @@ -18,9 +18,3 @@ services: - target: 3100 published: 3100 mode: host - - minio-01: - ports: - - target: 9001 - published: 9005 - mode: host diff --git a/monitoring/docker-compose.yml b/monitoring/docker-compose.yml index 2d4de27b..a60cb0fc 100644 --- a/monitoring/docker-compose.yml +++ b/monitoring/docker-compose.yml @@ -72,6 +72,7 @@ services: traefik: default: + prometheus: image: prom/prometheus:v2.38.0 user: root @@ -92,6 +93,7 @@ services: public: default: + cadvisor: image: gcr.io/cadvisor/cadvisor:v0.45.0 command: -docker_only @@ -125,6 +127,8 @@ services: - target: /etc/loki/loki-config.yml source: loki-config.yml command: -config.file=/etc/loki/loki-config.yml -config.expand-env=true + networks: + minio: deploy: labels: - prometheus-job-service=loki @@ -141,39 +145,6 @@ services: command: -config.file=/etc/promtail/promtail-config.yml deploy: mode: global - - minio-01: - image: quay.io/minio/minio:RELEASE.2022-10-24T18-35-07Z - entrypoint: sh - command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...${NUM_MINIO_SERVERS}}/data{1...2}' - environment: - MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER} - MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD} - MINIO_BROWSER_REDIRECT_URL: ${MINIO_BROWSER_REDIRECT_URL} - MINIO_SERVER_URL: http://localhost:9000 - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] - interval: 30s - timeout: 20s - retries: 3 - hostname: minio-01 - volumes: - - minio-01-data1:/data1 - - minio-01-data2:/data2 - deploy: - replicas: 1 - labels: - - traefik.enable=true - - traefik.docker.network=reverse-proxy-traefik_public - - traefik.http.routers.minio.rule=${DOMAIN_NAME_HOST_TRAEFIK} && PathPrefix(`/minio`) - - traefik.http.services.minio.loadbalancer.server.port=9001 - - traefik.http.middlewares.minio-stripprefix.stripprefix.prefixes=/minio - - traefik.http.routers.minio.middlewares=minio-stripprefix - networks: - reverse-proxy: - traefik: - default: - configs: grafana.ini: file: ./grafana/grafana.ini @@ -255,8 +226,7 @@ volumes: prometheus-data: grafana-data: loki-data: - minio-01-data1: - minio-01-data2: + networks: keycloak: @@ -271,4 +241,7 @@ networks: traefik: name: reverse-proxy-traefik_public external: true + minio: + name: minio_public + external: true default: diff --git a/monitoring/package-metadata.json b/monitoring/package-metadata.json index 82dc6b54..31691759 100644 --- a/monitoring/package-metadata.json +++ b/monitoring/package-metadata.json @@ -4,19 +4,14 @@ "description": "A package for monitoring the platform services", "type": "infrastructure", "version": "0.0.1", - "dependencies": [], + "dependencies": ["datalake"], "environmentVariables": { "GRAFANA_IMAGE": "grafana/grafana-oss:9.2.3", "LOKI_IMAGE": "grafana/loki:2.6.1", "PROMTAIL_IMAGE": "grafana/promtail:2.6.1", "PROMETHEUS_BACKUP_IMAGE": "prom/prometheus:v2.38.0", - "MINIO_IMAGE": "quay.io/minio/minio:RELEASE.2022-10-24T18-35-07Z", "PROMETHEUS_PLACEMENT": "node-1", "PROMETHEUS_BACKUP_PLACEMENT": "node-1", - "MINIO_01_PLACEMENT": "node-1", - "MINIO_02_PLACEMENT": "node-1", - "MINIO_03_PLACEMENT": "node-2", - "MINIO_04_PLACEMENT": "node-3", "GF_SECURITY_ADMIN_USER": "admin", "GF_SECURITY_ADMIN_PASSWORD": "dev_password_only", "GF_SMTP_ENABLED": "false",