fix: switch to prefork pool to avoid worker disconnects

README.md

@@ -2,9 +2,6 @@
 
 > HTTP wrapper around [openai/whisper](https://github.com/openai/whisper).
 
-
-> ⚠️ WARNING: The latest version has an unsolved bug and will not work until [issue #75](https://github.com/bellingcat/whisperbox-transcribe/issues/75) can be solved, help is welcome!
-
 ## Overview
 
 This project wraps OpenAI's `whisper` speech-to-text models with a HTTP API.

app/worker/main.py

@@ -67,11 +67,11 @@ def transcribe(self: TranscribeTask, job_id: UUID) -> None:
         job = session.query(models.Job).filter(models.Job.id == job_id).one_or_none()
 
         if job is None:
-            logger.warn("[{job.id}]: Received unknown job, abort.")
+            logger.warn("[unknown]: Received unknown job, abort.")
             return
 
         if job.status in [models.JobStatus.error, models.JobStatus.success]:
-            logger.warn("[{job.id}]: job has already been processed, abort.")
+            logger.warn(f"[{job.id}]: job has already been processed, abort.")
             return
 
         logger.debug(f"[{job.id}]: start processing {job.type} job.")

conf/rabbitmq.conf

@@ -1 +1,2 @@
 vm_memory_high_watermark.absolute = 192MB
+consumer_timeout = 31622400000

docker-compose.dev.yml

@@ -24,7 +24,7 @@ services:
       - "traefik.http.routers.web.rule=(Host(`${TRAEFIK_DOMAIN}`))"
 
   worker:
-    command: watchmedo auto-restart -d app/worker -p *.py --recursive celery -- --app=app.worker.main.celery worker --loglevel=info --concurrency=1 --pool solo
+    command: watchmedo auto-restart -d app/worker -p *.py --recursive celery -- --app=app.worker.main.celery worker --loglevel=info --concurrency=1 --pool prefork
     volumes:
       - ./:/etc/whisperbox-transcribe/
 

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "whisperbox-transcribe"
 description = ""
-version = "1.0.0"
+version = "1.0.1"
 
 dependencies=[
   "celery ==5.3.6",

worker.Dockerfile

@@ -28,4 +28,4 @@ RUN python download_models.py ${WHISPER_MODEL}
 
 COPY app ./app
 
-CMD celery --app=app.worker.main.celery worker --loglevel=info --concurrency=1 --pool=solo
+CMD celery --app=app.worker.main.celery worker --loglevel=info --pool=prefork --concurrency=1

worker.gpu.Dockerfile

@@ -36,4 +36,4 @@ ENV PATH /opt/venv/bin:$PATH
 COPY scripts/download_models.py .
 RUN python download_models.py ${WHISPER_MODEL}
 
-CMD celery --app=app.worker.main.celery worker --loglevel=info --concurrency=1 --pool=solo
+CMD celery --app=app.worker.main.celery worker --loglevel=info --concurrency=1 --pool=prefork