OPSEXP-2253 Add healthchecks and conditional service startup based on service health in docker compose

docker-compose/compose.yaml

@@ -52,14 +52,10 @@ services:
         -XX:MinRAMPercentage=50
         -XX:MaxRAMPercentage=80
     healthcheck:
-      test:
-        - CMD
-        - curl
-        - -f
-        - http://localhost:8080/alfresco/api/-default-/public/alfresco/versions/1/probes/-ready-
+      test: ["CMD", "curl", "-f", "http://localhost:8080/alfresco/api/-default-/public/alfresco/versions/1/probes/-ready-"]
       interval: 30s
       timeout: 3s
-      retries: 3
+      retries: 5
       start_period: 1m
     labels:
       - "traefik.enable=true"
@@ -84,8 +80,15 @@ services:
         http://shared-file-store:8099/alfresco/api/-default-/private/sfs/versions/1/file
     ports:
       - "8095:8095"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8095/actuator/health"]
+      interval: 60s
+      timeout: 10s
+      retries: 3
+      start_period: 20s
     depends_on:
-      - activemq
+      activemq:
+        condition: service_healthy
   transform-core-aio:
     image: alfresco/alfresco-transform-core-aio:5.1.4
     mem_limit: 1536m
@@ -98,8 +101,15 @@ services:
         http://shared-file-store:8099/alfresco/api/-default-/private/sfs/versions/1/file
     ports:
       - "8090:8090"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8090/ready"]
+      interval: 20s
+      timeout: 10s
+      retries: 3
+      start_period: 10s
     depends_on:
-      - activemq
+      activemq:
+        condition: service_healthy
   shared-file-store:
     image: quay.io/alfresco/alfresco-shared-file-store:4.1.4
     mem_limit: 512m
@@ -111,6 +121,12 @@ services:
       scheduler.cleanup.interval: 86400000
     ports:
       - "8099:8099"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8099/ready"]
+      interval: 20s
+      timeout: 10s
+      retries: 3
+      start_period: 10s
     volumes:
       - shared-file-store-volume:/tmp/Alfresco/sfs
   share:
@@ -128,6 +144,12 @@ services:
         -Dalfresco.port=8080
         -Dalfresco.context=alfresco
         -Dalfresco.protocol=http
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8080/share"]
+      interval: 20s
+      timeout: 5s
+      retries: 3
+      start_period: 15s
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.share.rule=PathPrefix(`/share`)"
@@ -145,6 +167,11 @@ services:
     command: postgres -c max_connections=300 -c log_min_messages=LOG
     ports:
       - "5432:5432"
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -d $$POSTGRES_DB -U $$POSTGRES_USER"]
+      interval: 10s
+      timeout: 3s
+      retries: 3
   elasticsearch:
     image: elasticsearch:7.17.3
     environment:
@@ -162,6 +189,12 @@ services:
     ports:
       - 9200:9200
       - 9300:9300
+    healthcheck:
+      test: ["CMD-SHELL", "curl -s -X GET http://localhost:9200/_cluster/health?pretty | grep status | grep -q '\\(green\\|yellow\\)'"]
+      interval: 10s
+      timeout: 10s
+      retries: 5
+      start_period: 30s
   kibana:
     image: kibana:7.17.3
     mem_limit: 1g
@@ -170,13 +203,20 @@ services:
     ports:
       - "5601:5601"
     depends_on:
-      - elasticsearch
+      elasticsearch:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD-SHELL", "curl -s -o /dev/null -w '%{http_code}' http://localhost:5601/status | grep -q '200'"]
+      interval: 10s
+      timeout: 10s
+      retries: 5
+      start_period: 30s
   search:
     image: quay.io/alfresco/alfresco-elasticsearch-live-indexing:4.1.0
     mem_limit: 1g
     depends_on:
       elasticsearch:
-        condition: service_started
+        condition: service_healthy
       search-reindexing:
         condition: service_completed_successfully
     environment:
@@ -186,6 +226,12 @@ services:
         http://shared-file-store:8099/alfresco/api/-default-/private/sfs/versions/1/file/
       SPRING_ACTIVEMQ_BROKERURL: nio://activemq:61616
       SPRING_ELASTICSEARCH_REST_URIS: http://elasticsearch:9200
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8080/actuator/health"]
+      interval: 20s
+      timeout: 20s
+      retries: 3
+      start_period: 60s
   search-reindexing:
     image: quay.io/alfresco/alfresco-elasticsearch-reindexing:4.1.0
     mem_limit: 1g
@@ -200,9 +246,9 @@ services:
       SPRING_ELASTICSEARCH_REST_URIS: http://elasticsearch:9200
     depends_on:
       postgres:
-        condition: service_started
+        condition: service_healthy
       elasticsearch:
-        condition: service_started
+        condition: service_healthy
       alfresco:
         condition: service_healthy
   activemq:
@@ -213,6 +259,11 @@ services:
       - "5672:5672" # AMQP
       - "61616:61616" # OpenWire
       - "61613:61613" # STOMP
+    healthcheck:
+      test: ["CMD", "/opt/activemq/bin/activemq", "query", "--objname", "type=Broker,brokerName=*,service=Health", "|", "grep", "Good"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
   audit-storage:
     image: quay.io/alfresco/alfresco-audit-storage:0.0.1-A8
     mem_limit: 512m
@@ -222,9 +273,17 @@ services:
       AUDIT_ENTRYSTORAGE_OPENSEARCH_CONNECTOR_USERNAME: admin
       AUDIT_ENTRYSTORAGE_OPENSEARCH_CONNECTOR_PASSWORD: admin
       AUDIT_EVENTINGESTION_URI: activemq:topic:alfresco.repo.event2
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8081/actuator/health/readiness"]
+      interval: 60s
+      timeout: 10s
+      retries: 3
+      start_period: 15s
     depends_on:
-      - activemq
-      - elasticsearch
+      activemq:
+        condition: service_healthy
+      elasticsearch:
+        condition: service_healthy
   digital-workspace:
     image: quay.io/alfresco/alfresco-digital-workspace:5.1.0
     mem_limit: 128m
@@ -235,6 +294,12 @@ services:
       APP_CONFIG_PLUGIN_PROCESS_SERVICE: "false"
       APP_CONFIG_PLUGIN_MICROSOFT_ONLINE: "false"
       APP_BASE_SHARE_URL: "http://localhost:8080/workspace/#/preview/s"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8080/"]
+      interval: 10s
+      timeout: 1s
+      retries: 3
+      start_period: 0s
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.adw.rule=PathPrefix(`/workspace`)"
@@ -250,6 +315,12 @@ services:
       APP_CONFIG_PROVIDER: "ECM"
       APP_CONFIG_AUTH_TYPE: "BASIC"
       BASE_PATH: ./
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8080/"]
+      interval: 10s
+      timeout: 1s
+      retries: 3
+      start_period: 0s
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.acc.rule=PathPrefix(`/control-center`) || PathPrefix(`/admin`)"
@@ -268,9 +339,17 @@ services:
       - "--entryPoints.traefik.address=:8888"
       - "--accesslog=true"
       - "--providers.docker.exposedByDefault=false"
+      - "--ping.entrypoint=web"
+      - "--ping=true"
     ports:
       - "8080:8080"
       - "8888:8888"
+    healthcheck:
+      test: ["CMD", "traefik", "healthcheck", "--ping"]
+      interval: 20s
+      timeout: 10s
+      retries: 3
+      start_period: 10s
     security_opt:
       - label=disable # Required for accessing the Docker socket on Selinux enabled systems
     volumes:
@@ -292,6 +371,12 @@ services:
         -XX:MaxRAMPercentage=80
     ports:
       - "9090:9090"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9090/alfresco/healthcheck"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 20s
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.syncservice.rule=PathPrefix(`/syncservice`)"