Refactor docker compose (#2288)

* WORKER_ env vars are only needed by the workers * fix variable name * remove obsolete env var * rename base services to the libs names * no need to create a different service * fix and refactor the docker compose files * fix syntax * reduce changes * fix typo * fix typo * remove port on worker
huggingface · Jan 15, 2024 · 3485673 · 3485673
1 parent a0a75be
commit 3485673
Show file tree

Hide file tree

Showing 4 changed files with 66 additions and 118 deletions.
diff --git a/tools/docker-compose-base.yml b/tools/docker-compose-base.yml
@@ -1,5 +1,5 @@
 services:
-  common:
+  libcommon:
     environment:
       # common
       COMMON_BLOCKED_DATASETS: ${COMMON_BLOCKED_DATASETS-}
@@ -19,19 +19,8 @@ services:
       # metrics
       METRICS_MONGO_URL: ${METRICS_MONGO_URL-mongodb://mongodb:${MONGO_PORT:-27017}} # use mongo container by default (with custom port)
       METRICS_MONGO_DATABASE: ${METRICS_MONGO_DATABASE-datasets_server_metrics}
-      # worker
-      WORKER_CONTENT_MAX_BYTES: ${WORKER_CONTENT_MAX_BYTES-10_000_000}
-      WORKER_KILL_LONG_JOB_INTERVAL_SECONDS: ${WORKER_KILL_LONG_JOB_INTERVAL_SECONDS-60}
-      WORKER_KILL_ZOMBIES_INTERVAL_SECONDS: ${WORKER_KILL_ZOMBIES_INTERVAL_SECONDS-600}
-      WORKER_MAX_DISK_USAGE_PCT: ${WORKER_MAX_DISK_USAGE_PCT-90}
-      WORKER_MAX_JOB_DURATION_SECONDS: ${WORKER_MAX_JOB_DURATION_SECONDS-1200}
-      WORKER_MAX_MISSING_HEARTBEATS: ${WORKER_MAX_MISSING_HEARTBEATS-5}
-      WORKER_MAX_LOAD_PCT: ${WORKER_MAX_LOAD_PCT-70}
-      WORKER_MAX_MEMORY_PCT: ${WORKER_MAX_MEMORY_PCT-80}
-      WORKER_SLEEP_SECONDS: ${WORKER_SLEEP_SECONDS-15}
-  api:
-    extends:
-      service: common
+  libapi:
+    extends: libcommon
     environment:
       NUMBA_CACHE_DIR: ${NUMBA_CACHE_DIR-/numba-cache}
       # service
@@ -42,12 +31,3 @@ services:
       API_HF_TIMEOUT_SECONDS: ${API_HF_TIMEOUT_SECONDS-0.2}
       API_MAX_AGE_LONG: ${API_MAX_AGE_LONG-120}
       API_MAX_AGE_SHORT: ${API_MAX_AGE_SHORT-10}
-
-  datasets-worker:
-    extends:
-      service: common
-    environment:
-      # datasets
-      DATASETS_BASED_HF_DATASETS_CACHE: ${HF_DATASETS_CACHE-/datasets-cache}
-      HF_MODULES_CACHE: ${HF_DATASETS_CACHE-/modules-cache}
-      NUMBA_CACHE_DIR: ${NUMBA_CACHE_DIR-/numba-cache}
diff --git a/tools/docker-compose-datasets-server.yml b/tools/docker-compose-datasets-server.yml
@@ -38,7 +38,7 @@ services:
       - descriptive-statistics:${DESCRIPTIVE_STATISTICS_CACHE_DIRECTORY-/stats-cache}:ro
     extends:
       file: docker-compose-base.yml
-      service: common
+      service: libapi
     environment:
       # service
       ADMIN_HF_ORGANIZATION: ${ADMIN_HF_ORGANIZATION-huggingface}
@@ -66,7 +66,6 @@ services:
         condition: service_healthy
     restart: always
     ports:
-      # for debug
       - ${ADMIN_UVICORN_PORT-8081}:${ADMIN_UVICORN_PORT-8081}
   api:
     build:
@@ -76,7 +75,7 @@ services:
       - storage:${STORAGE_DIRECTORY-/storage}:rw
     extends:
       file: docker-compose-base.yml
-      service: api
+      service: libapi
     environment:
       # prometheus
       PROMETHEUS_MULTIPROC_DIR: ${PROMETHEUS_MULTIPROC_DIR-}
@@ -90,7 +89,6 @@ services:
       CACHED_ASSETS_STORAGE_ROOT: ${CACHED_ASSETS_STORAGE_ROOT-/storage/cached-assets}
       CACHED_ASSETS_STORAGE_PROTOCOL: ${CACHED_ASSETS_STORAGE_PROTOCOL-file}
     ports:
-      # for debug
       - ${API_UVICORN_PORT-8080}:${API_UVICORN_PORT-8080}
     depends_on:
       mongodb:
@@ -102,7 +100,7 @@ services:
       dockerfile: services/rows/Dockerfile
     extends:
       file: docker-compose-base.yml
-      service: api
+      service: libapi
     volumes:
       - storage:${STORAGE_DIRECTORY-/storage}
       - parquet-metadata:${PARQUET_METADATA_STORAGE_DIRECTORY-/parquet_metadata}
@@ -134,7 +132,7 @@ services:
       dockerfile: services/search/Dockerfile
     extends:
       file: docker-compose-base.yml
-      service: api
+      service: libapi
     volumes:
       - storage:${STORAGE_DIRECTORY-/storage}
       - duckdb-index:${DUCKDB_INDEX_CACHE_DIRECTORY-/duckdb-index}
@@ -165,7 +163,7 @@ services:
       dockerfile: services/sse-api/Dockerfile
     extends:
       file: docker-compose-base.yml
-      service: api
+      service: libapi
     environment:
       # prometheus
       PROMETHEUS_MULTIPROC_DIR: ${PROMETHEUS_MULTIPROC_DIR-}
@@ -191,7 +189,7 @@ services:
       - descriptive-statistics:${DESCRIPTIVE_STATISTICS_CACHE_DIRECTORY-/stats-cache}:rw
     extends:
       file: docker-compose-base.yml
-      service: datasets-worker
+      service: libcommon
     environment:
       ASSETS_BASE_URL: "http://localhost:${PORT_REVERSE_PROXY-8000}/assets" # hard-coded to work with the reverse-proxy
       ASSETS_STORAGE_ROOT: ${ASSETS_STORAGE_ROOT-/storage/assets}
@@ -230,18 +228,29 @@ services:
       ROWS_INDEX_MAX_ARROW_DATA_IN_MEMORY: ${ROWS_INDEX_MAX_ARROW_DATA_IN_MEMORY-300_000_000}
       # prometheus
       PROMETHEUS_MULTIPROC_DIR: ${PROMETHEUS_MULTIPROC_DIR-}
-      # ^ note: the datasets cache is automatically added, so no need to add it here
       # uvicorn
       WORKER_UVICORN_HOSTNAME: 0.0.0.0 # required for docker compose
       WORKER_UVICORN_NUM_WORKERS: ${WORKER_UVICORN_NUM_WORKERS-2}
       WORKER_UVICORN_PORT: ${WORKER_UVICORN_PORT-8086}
+      # datasets
+      DATASETS_BASED_HF_DATASETS_CACHE: ${HF_DATASETS_CACHE-/datasets-cache}
+      HF_MODULES_CACHE: ${HF_DATASETS_CACHE-/modules-cache}
+      NUMBA_CACHE_DIR: ${NUMBA_CACHE_DIR-/numba-cache}
+      # worker
+      WORKER_CONTENT_MAX_BYTES: ${WORKER_CONTENT_MAX_BYTES-10_000_000}
+      WORKER_KILL_LONG_JOB_INTERVAL_SECONDS: ${WORKER_KILL_LONG_JOB_INTERVAL_SECONDS-60}
+      WORKER_KILL_ZOMBIES_INTERVAL_SECONDS: ${WORKER_KILL_ZOMBIES_INTERVAL_SECONDS-600}
+      WORKER_MAX_JOB_DURATION_SECONDS: ${WORKER_MAX_JOB_DURATION_SECONDS-1200}
+      WORKER_MAX_MISSING_HEARTBEATS: ${WORKER_MAX_MISSING_HEARTBEATS-5}
+      WORKER_MAX_LOAD_PCT: ${WORKER_MAX_LOAD_PCT-70}
+      WORKER_MAX_MEMORY_PCT: ${WORKER_MAX_MEMORY_PCT-80}
+      WORKER_SLEEP_SECONDS: ${WORKER_SLEEP_SECONDS-15}
+    ports:
+      - ${WORKER_UVICORN_PORT-8086}:${WORKER_UVICORN_PORT-8086}
     depends_on:
       mongodb:
         condition: service_healthy
     restart: always
-    ports:
-      # for debug
-      - ${WORKER_UVICORN_PORT-8086}:${WORKER_UVICORN_PORT-8086}
   mongodb:
     image: "mongo:6.0.9"
     ports:

diff --git a/tools/docker-compose-dev-base.yml b/tools/docker-compose-dev-base.yml
@@ -1,5 +1,5 @@
 services:
-  common:
+  libcommon:
     environment:
       # common
       COMMON_BLOCKED_DATASETS: ${COMMON_BLOCKED_DATASETS-}
@@ -19,65 +19,15 @@ services:
       # metrics
       METRICS_MONGO_URL: ${METRICS_MONGO_URL-mongodb://${DEV_MONGO_HOST-host.docker.internal}:${MONGO_PORT-27017}}
       METRICS_MONGO_DATABASE: ${METRICS_MONGO_DATABASE-datasets_server_metrics}
-      # worker
-      WORKER_CONTENT_MAX_BYTES: ${WORKER_CONTENT_MAX_BYTES-10_000_000}
-      WORKER_KILL_LONG_JOB_INTERVAL_SECONDS: ${WORKER_KILL_LONG_JOB_INTERVAL_SECONDS-60}
-      WORKER_KILL_ZOMBIES_INTERVAL_SECONDS: ${WORKER_KILL_ZOMBIES_INTERVAL_SECONDS-600}
-      WORKER_MAX_JOB_DURATION_SECONDS: ${WORKER_MAX_JOB_DURATION_SECONDS-1200}
-      WORKER_MAX_MISSING_HEARTBEATS: ${WORKER_MAX_MISSING_HEARTBEATS-5}
-      WORKER_MAX_LOAD_PCT: ${WORKER_MAX_LOAD_PCT-70}
-      WORKER_MAX_MEMORY_PCT: ${WORKER_MAX_MEMORY_PCT-80}
-      WORKER_SLEEP_SECONDS: ${WORKER_SLEEP_SECONDS-15}
     extra_hosts:
       - "host.docker.internal:host-gateway"
     # volumes to local source directory for development
     volumes:
       - ../libs/libcommon/src:/src/libs/libcommon/src
-  admin:
+    network_mode: ${DEV_NETWORK_MODE}
+  libapi:
     extends:
-      service: common
-    # volumes to local source directory for development
-    volumes:
-      - ../libs/libapi/src:/src/libs/libapi/src
-      - ../services/admin/src:/src/services/admin/src
-  api:
-    extends:
-      service: common
-    environment:
-      NUMBA_CACHE_DIR: ${NUMBA_CACHE_DIR-/numba-cache}
-      # service
-      API_HF_AUTH_PATH: ${API_HF_AUTH_PATH-/api/datasets/%s/auth-check}
-      API_HF_JWT_PUBLIC_KEY_URL: ${API_HF_JWT_PUBLIC_KEY_URL}
-      API_HF_JWT_ADDITIONAL_PUBLIC_KEYS: ${API_HF_JWT_ADDITIONAL_PUBLIC_KEYS}
-      API_HF_JWT_ALGORITHM: ${API_HF_JWT_ALGORITHM-EdDSA}
-      API_HF_TIMEOUT_SECONDS: ${API_HF_TIMEOUT_SECONDS-10.0}
-      API_MAX_AGE_LONG: ${API_MAX_AGE_LONG-120}
-      API_MAX_AGE_SHORT: ${API_MAX_AGE_SHORT-10}
-
-    # volumes to local source directory for development
-    volumes:
-      - ../libs/libapi/src:/src/libs/libapi/src
-      - ../services/api/src:/src/services/api/src
-  rows:
-    extends:
-      service: common
-    environment:
-      NUMBA_CACHE_DIR: ${NUMBA_CACHE_DIR-/numba-cache}
-    # volumes to local source directory for development
-    volumes:
-      - ../libs/libapi/src:/src/libs/libapi/src
-      - ../services/rows/src:/src/services/rows/src
-  search:
-    extends:
-      service: common
-    environment:
-    # volumes to local source directory for development
-    volumes:
-      - ../libs/libapi/src:/src/libs/libapi/src
-      - ../services/search/src:/src/services/search/src
-  sse-api:
-    extends:
-      service: common
+      service: libcommon
     environment:
       NUMBA_CACHE_DIR: ${NUMBA_CACHE_DIR-/numba-cache}
       # service
@@ -88,19 +38,6 @@ services:
       API_HF_TIMEOUT_SECONDS: ${API_HF_TIMEOUT_SECONDS-10.0}
       API_MAX_AGE_LONG: ${API_MAX_AGE_LONG-120}
       API_MAX_AGE_SHORT: ${API_MAX_AGE_SHORT-10}
-
     # volumes to local source directory for development
     volumes:
       - ../libs/libapi/src:/src/libs/libapi/src
-      - ../services/sse-api/src:/src/services/sse-api/src
-  datasets-worker:
-    extends:
-      service: common
-    environment:
-      # datasets
-      DATASETS_BASED_HF_DATASETS_CACHE: ${HF_DATASETS_CACHE-/datasets-cache}
-      HF_MODULES_CACHE: ${HF_DATASETS_CACHE-/modules-cache}
-      NUMBA_CACHE_DIR: ${NUMBA_CACHE_DIR-/numba-cache}
-    # volumes to local source directory for development
-    volumes:
-      - ../services/worker/src:/src/services/worker/src
diff --git a/tools/docker-compose-dev-datasets-server.yml b/tools/docker-compose-dev-datasets-server.yml
@@ -18,8 +18,6 @@ services:
       URL_ROWS: http://host.docker.internal:${ROWS_UVICORN_PORT-8082}
       URL_SEARCH: http://host.docker.internal:${SEARCH_UVICORN_PORT-8083}
       URL_SSE_API: http://host.docker.internal:${SSE_API_UVICORN_PORT-8085}
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
     depends_on:
       admin:
         condition: service_started
@@ -38,9 +36,11 @@ services:
       - duckdb-index:${DUCKDB_INDEX_CACHE_DIRECTORY-/duckdb-index}:ro
       - parquet-metadata:${PARQUET_METADATA_STORAGE_DIRECTORY-/parquet_metadata}:ro
       - descriptive-statistics:${DESCRIPTIVE_STATISTICS_CACHE_DIRECTORY-/stats-cache}:ro
+      # volumes to local source directory for development
+      - ../services/admin/src:/src/services/admin/src
     extends:
       file: docker-compose-dev-base.yml
-      service: admin
+      service: libapi
     environment:
       # service
       ADMIN_HF_ORGANIZATION: ${ADMIN_HF_ORGANIZATION-__DUMMY_DATASETS_SERVER_ADMINS_ORG__}
@@ -67,7 +67,6 @@ services:
       mongodb:
         condition: service_healthy
     restart: always
-    network_mode: ${DEV_NETWORK_MODE}
     ports:
       - ${ADMIN_UVICORN_PORT-8081}:${ADMIN_UVICORN_PORT-8081}
   api:
@@ -76,9 +75,11 @@ services:
       dockerfile: services/api/dev.Dockerfile
     volumes:
       - storage:${STORAGE_DIRECTORY-/storage}:rw
+      # volumes to local source directory for development
+      - ../services/api/src:/src/services/api/src
     extends:
       file: docker-compose-dev-base.yml
-      service: api
+      service: libapi
     environment:
       # prometheus
       PROMETHEUS_MULTIPROC_DIR: ${PROMETHEUS_MULTIPROC_DIR-}
@@ -91,7 +92,6 @@ services:
       ASSETS_STORAGE_PROTOCOL: ${ASSETS_STORAGE_PROTOCOL-file}
       CACHED_ASSETS_STORAGE_ROOT: ${CACHED_ASSETS_STORAGE_ROOT-/storage/cached-assets}
       CACHED_ASSETS_STORAGE_PROTOCOL: ${CACHED_ASSETS_STORAGE_PROTOCOL-file}
-    network_mode: ${DEV_NETWORK_MODE}
     ports:
       - ${API_UVICORN_PORT-8080}:${API_UVICORN_PORT-8080}
     depends_on:
@@ -104,10 +104,12 @@ services:
       dockerfile: services/rows/dev.Dockerfile
     extends:
       file: docker-compose-dev-base.yml
-      service: rows
+      service: libapi
     volumes:
       - storage:${STORAGE_DIRECTORY-/storage}
       - parquet-metadata:${PARQUET_METADATA_STORAGE_DIRECTORY-/parquet_metadata}
+      # volumes to local source directory for development
+      - ../services/rows/src:/src/services/rows/src
     environment:
       CACHED_ASSETS_BASE_URL: "http://localhost:${PORT_REVERSE_PROXY-8000}/cached-assets" # hard-coded to work with the reverse-proxy
       # storage
@@ -123,7 +125,6 @@ services:
       API_UVICORN_HOSTNAME: 0.0.0.0 # required for docker compose
       API_UVICORN_NUM_WORKERS: ${ROWS_UVICORN_NUM_WORKERS-2}
       API_UVICORN_PORT: ${ROWS_UVICORN_PORT-8082}
-    network_mode: ${DEV_NETWORK_MODE}
     ports:
       # for debug
       - ${ROWS_UVICORN_PORT-8082}:${ROWS_UVICORN_PORT-8082}
@@ -137,10 +138,12 @@ services:
       dockerfile: services/search/dev.Dockerfile
     extends:
       file: docker-compose-dev-base.yml
-      service: search
+      service: libapi
     volumes:
       - storage:${STORAGE_DIRECTORY-/storage}
       - duckdb-index:${DUCKDB_INDEX_CACHE_DIRECTORY-/duckdb-index}
+      # volumes to local source directory for development
+      - ../services/search/src:/src/services/search/src
     environment:
       CACHED_ASSETS_BASE_URL: "http://localhost:${PORT_REVERSE_PROXY-8000}/cached-assets" # hard-coded to work with the reverse-proxy
       # storage
@@ -155,7 +158,6 @@ services:
       API_UVICORN_HOSTNAME: 0.0.0.0 # required for docker compose
       API_UVICORN_NUM_WORKERS: ${SEARCH_UVICORN_NUM_WORKERS-2}
       API_UVICORN_PORT: ${SEARCH_UVICORN_PORT-8083}
-    network_mode: ${DEV_NETWORK_MODE}
     ports:
       # for debug
       - ${SEARCH_UVICORN_PORT-8083}:${SEARCH_UVICORN_PORT-8083}
@@ -169,15 +171,17 @@ services:
       dockerfile: services/sse-api/dev.Dockerfile
     extends:
       file: docker-compose-dev-base.yml
-      service: sse-api
+      service: libapi
+    volumes:
+      # volumes to local source directory for development
+      - ../services/sse-api/src:/src/services/sse-api/src
     environment:
       # prometheus
       PROMETHEUS_MULTIPROC_DIR: ${PROMETHEUS_MULTIPROC_DIR-}
       # uvicorn
       API_UVICORN_HOSTNAME: 0.0.0.0 # required for docker compose
       API_UVICORN_NUM_WORKERS: ${SSE_API_UVICORN_NUM_WORKERS-2}
       API_UVICORN_PORT: ${SSE_API_UVICORN_PORT-8085}
-    network_mode: ${DEV_NETWORK_MODE}
     ports:
       # for debug
       - ${SSE_API_UVICORN_PORT-8085}:${SSE_API_UVICORN_PORT-8085}
@@ -196,9 +200,11 @@ services:
       - parquet-metadata:${PARQUET_METADATA_STORAGE_DIRECTORY-/parquet_metadata}:rw
       - duckdb-index:${DUCKDB_INDEX_CACHE_DIRECTORY-/duckdb-index}:rw
       - descriptive-statistics:${DESCRIPTIVE_STATISTICS_CACHE_DIRECTORY-/stats-cache}:rw
+      # volumes to local source directory for development
+      - ../services/worker/src:/src/services/worker/src
     extends:
       file: docker-compose-dev-base.yml
-      service: datasets-worker
+      service: libcommon
     environment:
       ASSETS_BASE_URL: "http://localhost:${PORT_REVERSE_PROXY-8000}/assets" # hard-coded to work with the reverse-proxy
       ASSETS_STORAGE_ROOT: ${ASSETS_STORAGE_ROOT-/storage/assets}
@@ -241,9 +247,25 @@ services:
       WORKER_UVICORN_HOSTNAME: 0.0.0.0 # required for docker compose
       WORKER_UVICORN_NUM_WORKERS: ${WORKER_UVICORN_NUM_WORKERS-2}
       WORKER_UVICORN_PORT: ${WORKER_UVICORN_PORT-8086}
-    network_mode: ${DEV_NETWORK_MODE}
-    ports:
-      - ${WORKER_UVICORN_PORT-8086}:${WORKER_UVICORN_PORT-8086}
+      # datasets
+      DATASETS_BASED_HF_DATASETS_CACHE: ${HF_DATASETS_CACHE-/datasets-cache}
+      HF_MODULES_CACHE: ${HF_DATASETS_CACHE-/modules-cache}
+      NUMBA_CACHE_DIR: ${NUMBA_CACHE_DIR-/numba-cache}
+      # worker
+      WORKER_CONTENT_MAX_BYTES: ${WORKER_CONTENT_MAX_BYTES-10_000_000}
+      WORKER_KILL_LONG_JOB_INTERVAL_SECONDS: ${WORKER_KILL_LONG_JOB_INTERVAL_SECONDS-60}
+      WORKER_KILL_ZOMBIES_INTERVAL_SECONDS: ${WORKER_KILL_ZOMBIES_INTERVAL_SECONDS-600}
+      WORKER_MAX_JOB_DURATION_SECONDS: ${WORKER_MAX_JOB_DURATION_SECONDS-1200}
+      WORKER_MAX_MISSING_HEARTBEATS: ${WORKER_MAX_MISSING_HEARTBEATS-5}
+      WORKER_MAX_LOAD_PCT: ${WORKER_MAX_LOAD_PCT-70}
+      WORKER_MAX_MEMORY_PCT: ${WORKER_MAX_MEMORY_PCT-80}
+      WORKER_SLEEP_SECONDS: ${WORKER_SLEEP_SECONDS-15}
+    # ports:
+    #   - ${WORKER_UVICORN_PORT-8086}:${WORKER_UVICORN_PORT-8086}
+    # ^ disabling, since having 4 replicas of the worker service with the same port causes issue:
+    # Error response from daemon: driver failed programming external connectivity on endpoint
+    # dev-datasets-server-worker-2 (3619ef10b728504a43005f0381d4bc98da5f3a398475f8e3b305b1f504f40012):
+    # Bind for 0.0.0.0:8186 failed: port is already allocated
     depends_on:
       mongodb:
         condition: service_healthy