diff --git a/tools/docker-compose-base.yml b/tools/docker-compose-base.yml index 0e574f7e2e..bafc57c08e 100644 --- a/tools/docker-compose-base.yml +++ b/tools/docker-compose-base.yml @@ -1,5 +1,5 @@ services: - common: + libcommon: environment: # common COMMON_BLOCKED_DATASETS: ${COMMON_BLOCKED_DATASETS-} @@ -19,19 +19,8 @@ services: # metrics METRICS_MONGO_URL: ${METRICS_MONGO_URL-mongodb://mongodb:${MONGO_PORT:-27017}} # use mongo container by default (with custom port) METRICS_MONGO_DATABASE: ${METRICS_MONGO_DATABASE-datasets_server_metrics} - # worker - WORKER_CONTENT_MAX_BYTES: ${WORKER_CONTENT_MAX_BYTES-10_000_000} - WORKER_KILL_LONG_JOB_INTERVAL_SECONDS: ${WORKER_KILL_LONG_JOB_INTERVAL_SECONDS-60} - WORKER_KILL_ZOMBIES_INTERVAL_SECONDS: ${WORKER_KILL_ZOMBIES_INTERVAL_SECONDS-600} - WORKER_MAX_DISK_USAGE_PCT: ${WORKER_MAX_DISK_USAGE_PCT-90} - WORKER_MAX_JOB_DURATION_SECONDS: ${WORKER_MAX_JOB_DURATION_SECONDS-1200} - WORKER_MAX_MISSING_HEARTBEATS: ${WORKER_MAX_MISSING_HEARTBEATS-5} - WORKER_MAX_LOAD_PCT: ${WORKER_MAX_LOAD_PCT-70} - WORKER_MAX_MEMORY_PCT: ${WORKER_MAX_MEMORY_PCT-80} - WORKER_SLEEP_SECONDS: ${WORKER_SLEEP_SECONDS-15} - api: - extends: - service: common + libapi: + extends: libcommon environment: NUMBA_CACHE_DIR: ${NUMBA_CACHE_DIR-/numba-cache} # service @@ -42,12 +31,3 @@ services: API_HF_TIMEOUT_SECONDS: ${API_HF_TIMEOUT_SECONDS-0.2} API_MAX_AGE_LONG: ${API_MAX_AGE_LONG-120} API_MAX_AGE_SHORT: ${API_MAX_AGE_SHORT-10} - - datasets-worker: - extends: - service: common - environment: - # datasets - DATASETS_BASED_HF_DATASETS_CACHE: ${HF_DATASETS_CACHE-/datasets-cache} - HF_MODULES_CACHE: ${HF_DATASETS_CACHE-/modules-cache} - NUMBA_CACHE_DIR: ${NUMBA_CACHE_DIR-/numba-cache} diff --git a/tools/docker-compose-datasets-server.yml b/tools/docker-compose-datasets-server.yml index 493054921b..f0dbf1e18f 100644 --- a/tools/docker-compose-datasets-server.yml +++ b/tools/docker-compose-datasets-server.yml @@ -38,7 +38,7 @@ services: - descriptive-statistics:${DESCRIPTIVE_STATISTICS_CACHE_DIRECTORY-/stats-cache}:ro extends: file: docker-compose-base.yml - service: common + service: libapi environment: # service ADMIN_HF_ORGANIZATION: ${ADMIN_HF_ORGANIZATION-huggingface} @@ -66,7 +66,6 @@ services: condition: service_healthy restart: always ports: - # for debug - ${ADMIN_UVICORN_PORT-8081}:${ADMIN_UVICORN_PORT-8081} api: build: @@ -76,7 +75,7 @@ services: - storage:${STORAGE_DIRECTORY-/storage}:rw extends: file: docker-compose-base.yml - service: api + service: libapi environment: # prometheus PROMETHEUS_MULTIPROC_DIR: ${PROMETHEUS_MULTIPROC_DIR-} @@ -90,7 +89,6 @@ services: CACHED_ASSETS_STORAGE_ROOT: ${CACHED_ASSETS_STORAGE_ROOT-/storage/cached-assets} CACHED_ASSETS_STORAGE_PROTOCOL: ${CACHED_ASSETS_STORAGE_PROTOCOL-file} ports: - # for debug - ${API_UVICORN_PORT-8080}:${API_UVICORN_PORT-8080} depends_on: mongodb: @@ -102,7 +100,7 @@ services: dockerfile: services/rows/Dockerfile extends: file: docker-compose-base.yml - service: api + service: libapi volumes: - storage:${STORAGE_DIRECTORY-/storage} - parquet-metadata:${PARQUET_METADATA_STORAGE_DIRECTORY-/parquet_metadata} @@ -134,7 +132,7 @@ services: dockerfile: services/search/Dockerfile extends: file: docker-compose-base.yml - service: api + service: libapi volumes: - storage:${STORAGE_DIRECTORY-/storage} - duckdb-index:${DUCKDB_INDEX_CACHE_DIRECTORY-/duckdb-index} @@ -165,7 +163,7 @@ services: dockerfile: services/sse-api/Dockerfile extends: file: docker-compose-base.yml - service: api + service: libapi environment: # prometheus PROMETHEUS_MULTIPROC_DIR: ${PROMETHEUS_MULTIPROC_DIR-} @@ -191,7 +189,7 @@ services: - descriptive-statistics:${DESCRIPTIVE_STATISTICS_CACHE_DIRECTORY-/stats-cache}:rw extends: file: docker-compose-base.yml - service: datasets-worker + service: libcommon environment: ASSETS_BASE_URL: "http://localhost:${PORT_REVERSE_PROXY-8000}/assets" # hard-coded to work with the reverse-proxy ASSETS_STORAGE_ROOT: ${ASSETS_STORAGE_ROOT-/storage/assets} @@ -230,18 +228,29 @@ services: ROWS_INDEX_MAX_ARROW_DATA_IN_MEMORY: ${ROWS_INDEX_MAX_ARROW_DATA_IN_MEMORY-300_000_000} # prometheus PROMETHEUS_MULTIPROC_DIR: ${PROMETHEUS_MULTIPROC_DIR-} - # ^ note: the datasets cache is automatically added, so no need to add it here # uvicorn WORKER_UVICORN_HOSTNAME: 0.0.0.0 # required for docker compose WORKER_UVICORN_NUM_WORKERS: ${WORKER_UVICORN_NUM_WORKERS-2} WORKER_UVICORN_PORT: ${WORKER_UVICORN_PORT-8086} + # datasets + DATASETS_BASED_HF_DATASETS_CACHE: ${HF_DATASETS_CACHE-/datasets-cache} + HF_MODULES_CACHE: ${HF_DATASETS_CACHE-/modules-cache} + NUMBA_CACHE_DIR: ${NUMBA_CACHE_DIR-/numba-cache} + # worker + WORKER_CONTENT_MAX_BYTES: ${WORKER_CONTENT_MAX_BYTES-10_000_000} + WORKER_KILL_LONG_JOB_INTERVAL_SECONDS: ${WORKER_KILL_LONG_JOB_INTERVAL_SECONDS-60} + WORKER_KILL_ZOMBIES_INTERVAL_SECONDS: ${WORKER_KILL_ZOMBIES_INTERVAL_SECONDS-600} + WORKER_MAX_JOB_DURATION_SECONDS: ${WORKER_MAX_JOB_DURATION_SECONDS-1200} + WORKER_MAX_MISSING_HEARTBEATS: ${WORKER_MAX_MISSING_HEARTBEATS-5} + WORKER_MAX_LOAD_PCT: ${WORKER_MAX_LOAD_PCT-70} + WORKER_MAX_MEMORY_PCT: ${WORKER_MAX_MEMORY_PCT-80} + WORKER_SLEEP_SECONDS: ${WORKER_SLEEP_SECONDS-15} + ports: + - ${WORKER_UVICORN_PORT-8086}:${WORKER_UVICORN_PORT-8086} depends_on: mongodb: condition: service_healthy restart: always - ports: - # for debug - - ${WORKER_UVICORN_PORT-8086}:${WORKER_UVICORN_PORT-8086} mongodb: image: "mongo:6.0.9" ports: diff --git a/tools/docker-compose-dev-base.yml b/tools/docker-compose-dev-base.yml index 1a3c61eed7..b9bc72e685 100644 --- a/tools/docker-compose-dev-base.yml +++ b/tools/docker-compose-dev-base.yml @@ -1,5 +1,5 @@ services: - common: + libcommon: environment: # common COMMON_BLOCKED_DATASETS: ${COMMON_BLOCKED_DATASETS-} @@ -19,65 +19,15 @@ services: # metrics METRICS_MONGO_URL: ${METRICS_MONGO_URL-mongodb://${DEV_MONGO_HOST-host.docker.internal}:${MONGO_PORT-27017}} METRICS_MONGO_DATABASE: ${METRICS_MONGO_DATABASE-datasets_server_metrics} - # worker - WORKER_CONTENT_MAX_BYTES: ${WORKER_CONTENT_MAX_BYTES-10_000_000} - WORKER_KILL_LONG_JOB_INTERVAL_SECONDS: ${WORKER_KILL_LONG_JOB_INTERVAL_SECONDS-60} - WORKER_KILL_ZOMBIES_INTERVAL_SECONDS: ${WORKER_KILL_ZOMBIES_INTERVAL_SECONDS-600} - WORKER_MAX_JOB_DURATION_SECONDS: ${WORKER_MAX_JOB_DURATION_SECONDS-1200} - WORKER_MAX_MISSING_HEARTBEATS: ${WORKER_MAX_MISSING_HEARTBEATS-5} - WORKER_MAX_LOAD_PCT: ${WORKER_MAX_LOAD_PCT-70} - WORKER_MAX_MEMORY_PCT: ${WORKER_MAX_MEMORY_PCT-80} - WORKER_SLEEP_SECONDS: ${WORKER_SLEEP_SECONDS-15} extra_hosts: - "host.docker.internal:host-gateway" # volumes to local source directory for development volumes: - ../libs/libcommon/src:/src/libs/libcommon/src - admin: + network_mode: ${DEV_NETWORK_MODE} + libapi: extends: - service: common - # volumes to local source directory for development - volumes: - - ../libs/libapi/src:/src/libs/libapi/src - - ../services/admin/src:/src/services/admin/src - api: - extends: - service: common - environment: - NUMBA_CACHE_DIR: ${NUMBA_CACHE_DIR-/numba-cache} - # service - API_HF_AUTH_PATH: ${API_HF_AUTH_PATH-/api/datasets/%s/auth-check} - API_HF_JWT_PUBLIC_KEY_URL: ${API_HF_JWT_PUBLIC_KEY_URL} - API_HF_JWT_ADDITIONAL_PUBLIC_KEYS: ${API_HF_JWT_ADDITIONAL_PUBLIC_KEYS} - API_HF_JWT_ALGORITHM: ${API_HF_JWT_ALGORITHM-EdDSA} - API_HF_TIMEOUT_SECONDS: ${API_HF_TIMEOUT_SECONDS-10.0} - API_MAX_AGE_LONG: ${API_MAX_AGE_LONG-120} - API_MAX_AGE_SHORT: ${API_MAX_AGE_SHORT-10} - - # volumes to local source directory for development - volumes: - - ../libs/libapi/src:/src/libs/libapi/src - - ../services/api/src:/src/services/api/src - rows: - extends: - service: common - environment: - NUMBA_CACHE_DIR: ${NUMBA_CACHE_DIR-/numba-cache} - # volumes to local source directory for development - volumes: - - ../libs/libapi/src:/src/libs/libapi/src - - ../services/rows/src:/src/services/rows/src - search: - extends: - service: common - environment: - # volumes to local source directory for development - volumes: - - ../libs/libapi/src:/src/libs/libapi/src - - ../services/search/src:/src/services/search/src - sse-api: - extends: - service: common + service: libcommon environment: NUMBA_CACHE_DIR: ${NUMBA_CACHE_DIR-/numba-cache} # service @@ -88,19 +38,6 @@ services: API_HF_TIMEOUT_SECONDS: ${API_HF_TIMEOUT_SECONDS-10.0} API_MAX_AGE_LONG: ${API_MAX_AGE_LONG-120} API_MAX_AGE_SHORT: ${API_MAX_AGE_SHORT-10} - # volumes to local source directory for development volumes: - ../libs/libapi/src:/src/libs/libapi/src - - ../services/sse-api/src:/src/services/sse-api/src - datasets-worker: - extends: - service: common - environment: - # datasets - DATASETS_BASED_HF_DATASETS_CACHE: ${HF_DATASETS_CACHE-/datasets-cache} - HF_MODULES_CACHE: ${HF_DATASETS_CACHE-/modules-cache} - NUMBA_CACHE_DIR: ${NUMBA_CACHE_DIR-/numba-cache} - # volumes to local source directory for development - volumes: - - ../services/worker/src:/src/services/worker/src diff --git a/tools/docker-compose-dev-datasets-server.yml b/tools/docker-compose-dev-datasets-server.yml index 111c481a77..222c7fca4d 100644 --- a/tools/docker-compose-dev-datasets-server.yml +++ b/tools/docker-compose-dev-datasets-server.yml @@ -18,8 +18,6 @@ services: URL_ROWS: http://host.docker.internal:${ROWS_UVICORN_PORT-8082} URL_SEARCH: http://host.docker.internal:${SEARCH_UVICORN_PORT-8083} URL_SSE_API: http://host.docker.internal:${SSE_API_UVICORN_PORT-8085} - extra_hosts: - - "host.docker.internal:host-gateway" depends_on: admin: condition: service_started @@ -38,9 +36,11 @@ services: - duckdb-index:${DUCKDB_INDEX_CACHE_DIRECTORY-/duckdb-index}:ro - parquet-metadata:${PARQUET_METADATA_STORAGE_DIRECTORY-/parquet_metadata}:ro - descriptive-statistics:${DESCRIPTIVE_STATISTICS_CACHE_DIRECTORY-/stats-cache}:ro + # volumes to local source directory for development + - ../services/admin/src:/src/services/admin/src extends: file: docker-compose-dev-base.yml - service: admin + service: libapi environment: # service ADMIN_HF_ORGANIZATION: ${ADMIN_HF_ORGANIZATION-__DUMMY_DATASETS_SERVER_ADMINS_ORG__} @@ -67,7 +67,6 @@ services: mongodb: condition: service_healthy restart: always - network_mode: ${DEV_NETWORK_MODE} ports: - ${ADMIN_UVICORN_PORT-8081}:${ADMIN_UVICORN_PORT-8081} api: @@ -76,9 +75,11 @@ services: dockerfile: services/api/dev.Dockerfile volumes: - storage:${STORAGE_DIRECTORY-/storage}:rw + # volumes to local source directory for development + - ../services/api/src:/src/services/api/src extends: file: docker-compose-dev-base.yml - service: api + service: libapi environment: # prometheus PROMETHEUS_MULTIPROC_DIR: ${PROMETHEUS_MULTIPROC_DIR-} @@ -91,7 +92,6 @@ services: ASSETS_STORAGE_PROTOCOL: ${ASSETS_STORAGE_PROTOCOL-file} CACHED_ASSETS_STORAGE_ROOT: ${CACHED_ASSETS_STORAGE_ROOT-/storage/cached-assets} CACHED_ASSETS_STORAGE_PROTOCOL: ${CACHED_ASSETS_STORAGE_PROTOCOL-file} - network_mode: ${DEV_NETWORK_MODE} ports: - ${API_UVICORN_PORT-8080}:${API_UVICORN_PORT-8080} depends_on: @@ -104,10 +104,12 @@ services: dockerfile: services/rows/dev.Dockerfile extends: file: docker-compose-dev-base.yml - service: rows + service: libapi volumes: - storage:${STORAGE_DIRECTORY-/storage} - parquet-metadata:${PARQUET_METADATA_STORAGE_DIRECTORY-/parquet_metadata} + # volumes to local source directory for development + - ../services/rows/src:/src/services/rows/src environment: CACHED_ASSETS_BASE_URL: "http://localhost:${PORT_REVERSE_PROXY-8000}/cached-assets" # hard-coded to work with the reverse-proxy # storage @@ -123,7 +125,6 @@ services: API_UVICORN_HOSTNAME: 0.0.0.0 # required for docker compose API_UVICORN_NUM_WORKERS: ${ROWS_UVICORN_NUM_WORKERS-2} API_UVICORN_PORT: ${ROWS_UVICORN_PORT-8082} - network_mode: ${DEV_NETWORK_MODE} ports: # for debug - ${ROWS_UVICORN_PORT-8082}:${ROWS_UVICORN_PORT-8082} @@ -137,10 +138,12 @@ services: dockerfile: services/search/dev.Dockerfile extends: file: docker-compose-dev-base.yml - service: search + service: libapi volumes: - storage:${STORAGE_DIRECTORY-/storage} - duckdb-index:${DUCKDB_INDEX_CACHE_DIRECTORY-/duckdb-index} + # volumes to local source directory for development + - ../services/search/src:/src/services/search/src environment: CACHED_ASSETS_BASE_URL: "http://localhost:${PORT_REVERSE_PROXY-8000}/cached-assets" # hard-coded to work with the reverse-proxy # storage @@ -155,7 +158,6 @@ services: API_UVICORN_HOSTNAME: 0.0.0.0 # required for docker compose API_UVICORN_NUM_WORKERS: ${SEARCH_UVICORN_NUM_WORKERS-2} API_UVICORN_PORT: ${SEARCH_UVICORN_PORT-8083} - network_mode: ${DEV_NETWORK_MODE} ports: # for debug - ${SEARCH_UVICORN_PORT-8083}:${SEARCH_UVICORN_PORT-8083} @@ -169,7 +171,10 @@ services: dockerfile: services/sse-api/dev.Dockerfile extends: file: docker-compose-dev-base.yml - service: sse-api + service: libapi + volumes: + # volumes to local source directory for development + - ../services/sse-api/src:/src/services/sse-api/src environment: # prometheus PROMETHEUS_MULTIPROC_DIR: ${PROMETHEUS_MULTIPROC_DIR-} @@ -177,7 +182,6 @@ services: API_UVICORN_HOSTNAME: 0.0.0.0 # required for docker compose API_UVICORN_NUM_WORKERS: ${SSE_API_UVICORN_NUM_WORKERS-2} API_UVICORN_PORT: ${SSE_API_UVICORN_PORT-8085} - network_mode: ${DEV_NETWORK_MODE} ports: # for debug - ${SSE_API_UVICORN_PORT-8085}:${SSE_API_UVICORN_PORT-8085} @@ -196,9 +200,11 @@ services: - parquet-metadata:${PARQUET_METADATA_STORAGE_DIRECTORY-/parquet_metadata}:rw - duckdb-index:${DUCKDB_INDEX_CACHE_DIRECTORY-/duckdb-index}:rw - descriptive-statistics:${DESCRIPTIVE_STATISTICS_CACHE_DIRECTORY-/stats-cache}:rw + # volumes to local source directory for development + - ../services/worker/src:/src/services/worker/src extends: file: docker-compose-dev-base.yml - service: datasets-worker + service: libcommon environment: ASSETS_BASE_URL: "http://localhost:${PORT_REVERSE_PROXY-8000}/assets" # hard-coded to work with the reverse-proxy ASSETS_STORAGE_ROOT: ${ASSETS_STORAGE_ROOT-/storage/assets} @@ -241,9 +247,25 @@ services: WORKER_UVICORN_HOSTNAME: 0.0.0.0 # required for docker compose WORKER_UVICORN_NUM_WORKERS: ${WORKER_UVICORN_NUM_WORKERS-2} WORKER_UVICORN_PORT: ${WORKER_UVICORN_PORT-8086} - network_mode: ${DEV_NETWORK_MODE} - ports: - - ${WORKER_UVICORN_PORT-8086}:${WORKER_UVICORN_PORT-8086} + # datasets + DATASETS_BASED_HF_DATASETS_CACHE: ${HF_DATASETS_CACHE-/datasets-cache} + HF_MODULES_CACHE: ${HF_DATASETS_CACHE-/modules-cache} + NUMBA_CACHE_DIR: ${NUMBA_CACHE_DIR-/numba-cache} + # worker + WORKER_CONTENT_MAX_BYTES: ${WORKER_CONTENT_MAX_BYTES-10_000_000} + WORKER_KILL_LONG_JOB_INTERVAL_SECONDS: ${WORKER_KILL_LONG_JOB_INTERVAL_SECONDS-60} + WORKER_KILL_ZOMBIES_INTERVAL_SECONDS: ${WORKER_KILL_ZOMBIES_INTERVAL_SECONDS-600} + WORKER_MAX_JOB_DURATION_SECONDS: ${WORKER_MAX_JOB_DURATION_SECONDS-1200} + WORKER_MAX_MISSING_HEARTBEATS: ${WORKER_MAX_MISSING_HEARTBEATS-5} + WORKER_MAX_LOAD_PCT: ${WORKER_MAX_LOAD_PCT-70} + WORKER_MAX_MEMORY_PCT: ${WORKER_MAX_MEMORY_PCT-80} + WORKER_SLEEP_SECONDS: ${WORKER_SLEEP_SECONDS-15} + # ports: + # - ${WORKER_UVICORN_PORT-8086}:${WORKER_UVICORN_PORT-8086} + # ^ disabling, since having 4 replicas of the worker service with the same port causes issue: + # Error response from daemon: driver failed programming external connectivity on endpoint + # dev-datasets-server-worker-2 (3619ef10b728504a43005f0381d4bc98da5f3a398475f8e3b305b1f504f40012): + # Bind for 0.0.0.0:8186 failed: port is already allocated depends_on: mongodb: condition: service_healthy