Merge branch 'main' into CU-86bygcqxd_Images-Placement-for-DataStore-…

…Dashboards-Monitoring

.env.cluster

@@ -27,6 +27,7 @@ HAPI_DB_SET=pgpool-1:5432,pgpool-2:5432,pgpool-3:5432
 
 # Sante Datastore - Sante MPI
 SANTEMPI_REPMGR_PARTNER_NODES=santempi-psql-1,santempi-psql-2,santempi-psql-3
+SANTEMPI_DB_REP_SET=pgpool-1:5432,pgpool-2:5432,pgpool-3:5432
 
 # Reverse Proxy - Nginx
 REVERSE_PROXY_INSTANCES=3
@@ -79,6 +80,7 @@ KC_JEMPI_ROOT_URL=https://jempi-web.domain
 KC_SUPERSET_ROOT_URL=https://superset.domain
 KC_OPENHIM_ROOT_URL=https://openhimconsole.domain
 GF_SERVER_DOMAIN=grafana.domain
+KC_POSTGRES_REPLICA_SET=pgpool-1:5432,pgpool-2:5432,pgpool-3:5432
 
 # Client Registry - JeMPI
 JEMPI_WEB_INSTANCES=3
@@ -100,3 +102,8 @@ ES_MEMORY_LIMIT=20G
 LOGSTASH_MEMORY_LIMIT=8G
 KAFKA_MEMORY_LIMIT=8G
 KAFDROP_MEMORY_LIMIT=500M
+
+# PG Pool
+# The postgres users have to be specified so that they can be added to the pg_pool authentication interceptor
+PGPOOL_POSTGRES_CUSTOM_USERS=hapi,keycloak
+PGPOOL_POSTGRES_CUSTOM_PASSWORDS=instant101,instant101

Dockerfile

@@ -1,4 +1,4 @@
-FROM openhie/package-base:2.2.0
+FROM openhie/package-base:2.3.1
 
 # Install yq
 RUN curl -L https://github.com/mikefarah/yq/releases/download/v4.23.1/yq_linux_amd64 -o /usr/bin/yq

client-registry-santempi/docker-compose.yml

@@ -8,13 +8,14 @@ services:
       public:
       reverse-proxy:
       default:
+      postgres:
     environment:
       - SDB_FEATURE=LOG;DATA_POLICY;AUDIT_REPO;ADO;PUBSUB_ADO;RAMCACHE;SEC;SWAGGER;OPENID;FHIR;HL7;HDSI;AMI;BIS;MDM;MATCHING;ADO_FTS
       - SDB_MATCHING_MODE=WEIGHTED
       - SDB_MDM_RESOURCE=Patient=org.santedb.matching.patient.default
       - SDB_MDM_AUTO_MERGE=false
-      - SDB_DB_MAIN=server=${SANTEMPI_REPMGR_PRIMARY_HOST};port=5432; database=santedb; user id=${SANTEMPI_POSTGRESQL_USERNAME}; password=${SANTEMPI_POSTGRESQL_PASSWORD}; pooling=true; MinPoolSize=5; MaxPoolSize=15; Timeout=60;
-      - SDB_DB_AUDIT=server=${SANTEMPI_REPMGR_PRIMARY_HOST};port=5432; database=auditdb; user id=${SANTEMPI_POSTGRESQL_USERNAME}; password=${SANTEMPI_POSTGRESQL_PASSWORD}; pooling=true; MinPoolSize=5; MaxPoolSize=15; Timeout=60;
+      - SDB_DB_MAIN=server=${SANTEMPI_DB_REP_SET};port=5432; database=santedb; user id=${POSTGRESQL_USERNAME}; password=${POSTGRESQL_PASSWORD}; pooling=true; MinPoolSize=5; MaxPoolSize=15; Timeout=60;
+      - SDB_DB_AUDIT=server=${SANTEMPI_DB_REP_SET};port=5432; database=auditdb; user id=${POSTGRESQL_USERNAME}; password=${POSTGRESQL_PASSWORD}; pooling=true; MinPoolSize=5; MaxPoolSize=15; Timeout=60;
       - SDB_DB_MAIN_PROVIDER=Npgsql
       - SDB_DB_AUDIT_PROVIDER=Npgsql
       - SDB_DATA_POLICY_ACTION=HIDE
@@ -46,3 +47,6 @@ networks:
     name: reverse-proxy_public
     external: true
   default:
+  postgres:
+    name: postgres_public
+    external: true

client-registry-santempi/package-metadata.json

@@ -4,13 +4,11 @@
   "description": "A patient matching and deduplicater for the platform",
   "type": "infrastructure",
   "version": "0.0.1",
-  "dependencies": [],
+  "dependencies": ["database-postgres"],
   "environmentVariables": {
     "SANTEMPI_INSTANCES": "1",
-    "SANTE_POSTGRES_PASSWORD": "postgres",
-    "SANTEMPI_POSTGRESQL_PASSWORD": "SanteDB123",
-    "SANTEMPI_POSTGRESQL_USERNAME": "santedb",
-    "SANTEMPI_REPMGR_PRIMARY_HOST": "santempi-psql-1",
-    "SANTEMPI_REPMGR_PARTNER_NODES": "santempi-psql-1"
+    "POSTGRESQL_PASSWORD": "instant101",
+    "POSTGRESQL_USERNAME": "postgres",
+    "SANTEMPI_DB_REP_SET": "postgres-1"
   }
 }

client-registry-santempi/swarm.sh

@@ -32,24 +32,17 @@ function import_sources() {
 }
 
 function initialize_package() {
-  local postgres_cluster_compose_filename=""
-  local postgres_dev_compose_filename=""
   local sante_mpi_dev_compose_filename=""
 
   if [[ "$MODE" == "dev" ]]; then
     log info "Running package in DEV mode"
-    postgres_dev_compose_filename="docker-compose-postgres.dev.yml"
     sante_mpi_dev_compose_filename="docker-compose.dev.yml"
   else
     log info "Running package in PROD mode"
   fi
 
-  if [[ "${CLUSTERED_MODE}" == "true" ]]; then
-    postgres_cluster_compose_filename="docker-compose-postgres.cluster.yml"
-  fi
-
   (
-    docker::deploy_service "$STACK" "${COMPOSE_FILE_PATH}" "docker-compose-postgres.yml" "$postgres_cluster_compose_filename" "$postgres_dev_compose_filename"
+    docker::await_service_status "postgres" "postgres-1" "Running"
 
     docker::deploy_service "$STACK" "${COMPOSE_FILE_PATH}" "docker-compose.yml" "$sante_mpi_dev_compose_filename"
   ) ||

config.yaml

@@ -23,6 +23,7 @@ packages:
   - client-registry-jempi
   - identity-access-manager-keycloak
   - openhim-mapping-mediator
+  - database-postgres
 
 profiles:
   - name: cdr-dw

database-postgres/README.md

@@ -0,0 +1,123 @@
+
+# Instant OpenHIE Postgres database - docker-swarm
+
+## Getting Started
+
+> **The below instructions are only to be used for starting up the Postgres manually for local testing outside of the usual Instant OpenHIE start instructions.**
+
+Proceed with care. This very manual deployment can get complicated.
+For the regular start up, please see the [README.md](../../README.md).
+
+### Prerequisites
+
+Ensure that docker is installed. For details on how to install docker click [here](https://linuxize.com/post/how-to-install-and-use-docker-compose-on-ubuntu-18-04/).
+For installing docker click [here](https://linuxize.com/post/how-to-install-and-use-docker-on-ubuntu-18-04/).
+
+For our compose scripts to work, one needs to be able to run docker commands without the `sudo` preface. You can configure your system to run without needing the `sudo` preface by running the following command
+
+```bash
+./configure-docker.sh
+```
+
+### Start Up Postgres Service
+
+From the instant root directory, run the following command to start up the fhir data store.
+
+```bash
+./database-postgres/swarm.sh init
+```
+
+To take down the service run:
+
+```bash
+./database-postgres/swarm.sh destroy
+```
+
+To shut down the services run:
+
+```bash
+./database-postgres/swarm.sh down
+```
+
+To start the services when they have been stopped run:
+
+```bash
+./database-postgres/swarm.sh up
+```
+
+To run in dev mode in which the ports are exposed pass the flag `--dev` as done below
+
+```bash
+./database-postgres/swarm.sh init --dev
+```
+
+## Accessing the services
+
+Thid service is accessible on port 5432 when deployed in dev mode.
+
+## Backups
+
+> This section assumes postgres backups are made using `pg_basebackup`
+
+### Postgres
+
+To enable backups, ensure that you have created the Hapi FHIR bind mount directory (eg./backup)
+
+## Disaster Recovery
+
+> NB!!! DO NOT UNTAR OR EDIT THE FILE PERMISSIONS OF THE POSTGRES BACKUP FILE
+
+### Postgres
+
+Preliminary steps:
+
+1. Do a `destroy` of `database-postgres` using the CLI binary (./instant-linux for linux)
+1. Make sure the Postgres volumes on nodes other than the swarm leader have been removed as well! You will need to ssh into each server and manually remove them.
+1. Do an `init` of `database-postgres` using the CLI binary
+
+After running the premilinary steps, run the following commands on the node hosting the Postgres leader:
+
+> NOTE: The value of the `REPMGR_PRIMARY_HOST` variable in your .env file indicates the Postgres leader
+
+1. Retrieve the Postgres leader's container-ID using `docker ps -a`, hereafter called `postgres_leader_container_id`
+1. Do `docker exec -t <postgres_leader_container_id> pg_ctl stop -D /bitnami/postgresql/data`
+1. Wait for the Postgres leader container to die and start up again... monitor this using `docker ps -a`
+1. Do `docker rm <postgres_leader_container_id>`
+1. Retrieve the new Postgres leader's container-ID using `docker ps -a`, be weary to not use the old `postgres_leader_container_id`
+1. Retrieve the Postgres backup file's name as an absolute path (/backups/postgresql_xxxxxxxxxx), hereafter called `backup_file`
+1. Run the following commands in the order listed :
+    ```sh
+    # Stop the server running in the container
+    docker exec -t <postgres_leader_container_id> pg_ctl stop -D /bitnami/postgresql/data
+
+    # Clear the contents of /bitnami/postgresql/data
+    docker exec -t --user root <postgres_leader_container_id> sh -c 'cd /bitnami/postgresql/data && rm -rf $(ls)'
+
+    # Copy over the base.tar file
+    sudo docker cp <backup_file>/base.tar <postgres_leader_container_id>:/bitnami/postgresql
+
+    # Extract the base.tar file
+    docker exec -t --user root <postgres_leader_container_id> sh -c 'tar -xf /bitnami/postgresql/base.tar --directory=/bitnami/postgresql/data'
+
+    # Copy over the pg_wal.tar file
+    sudo docker cp <backup_file>/pg_wal.tar <postgres_leader_container_id>:/bitnami/postgresql
+
+    # Extract pg_wal.tar
+    docker exec -t --user root <postgres_leader_container_id> sh -c 'tar -xf /bitnami/postgresql/pg_wal.tar --directory=/bitnami/postgresql/data/pg_wal'
+
+    # Copy conf dir over
+    docker exec -t --user root <postgres_leader_container_id> sh -c 'cp -r /bitnami/postgresql/conf/. /bitnami/postgresql/data'
+
+    # Set pg_wal.tar permissions
+    docker exec -t --user root <postgres_leader_container_id> sh -c 'cd /bitnami/postgresql/data/pg_wal && chown -v 1001 $(ls)'
+
+    # Start the server
+    docker exec -t <postgres_leader_container_id> pg_ctl start -D /bitnami/postgresql/data
+    ```
+1. Do a `down` of `database-postgres` using the CLI binary
+1. Wait for the `down` operation to complete
+1. Do an `init` of `database-postgres` using the CLI binary
+
+Postgres should now be recovered
+
+> Note: after performing the data recovery, it is possible to get an error from services using postgres (eg 500 internal server error for Hapi-fhir) while the data is still being replicated across the cluster. Wait a minute and try again.