Merge pull request #9 from jokester/remove-ocr-task

disable `ocr` task

CONTRIBUTION.md

@@ -0,0 +1,21 @@
+# CONTRIBUTION
+
+## Structure
+
+
+## HOWTOs
+
+### init venv for development
+
+```
+$ python3.10 -mvenv venv
+$ venv/bin/pip install -r requirements.txt
+```
+
+### lint + format code
+
+```
+$ venv/bin/ruff .
+```
+
+

README.md

@@ -8,7 +8,7 @@
 
 ## 安装步骤
 
-1. 安装 Python 3.8.13 版本以上，3.10 版本以下，推荐 `3.9.2`
+1. 安装 Python 3.10 版本
 2. 依赖环境 MangoDB、Erlang、RabbitMQ
 3. `pip install -r requirements.txt` （这一步如果 Windows 有报错，请在环境变量里面加 `PYTHONUTF8=1` ）
 4. 以 `/config.py` 为模板创建 `/configs/dev.py` 用于开发（此目录已被 git ignore）

app/__init__.py

@@ -1,7 +1,6 @@
 import os
 
 from celery import Celery
-from celery.signals import worker_shutting_down
 from flask import Flask, g, request
 from flask_apikit import APIKit
 from flask_babel import Babel
@@ -118,32 +117,21 @@ def create_app():
     return app
 
 
-CELERY_ABOUT_TO_SHUTDOWN_FLAG = "CELERY_ABOUT_TO_SHUTDOWN_FLAG"
-
-
-def delete_about_to_shutdown_flag():
-    try:
-        os.rmdir(CELERY_ABOUT_TO_SHUTDOWN_FLAG)
-    except Exception:
-        pass
-
-
-def create_celery():
-    delete_about_to_shutdown_flag()
+def create_celery() -> Celery:
     # 为celery创建app
     app = Flask(__name__)
     app.config.from_envvar(
         config_path_env
     )  # 获取配置文件,仅从环境变量读取,均需要配置环境变量
     # 通过app配置创建celery实例
-    celery = Celery(
+    created = Celery(
         app.name,
         broker=app.config["CELERY_BROKER_URL"],
         backend=app.config["CELERY_BACKEND_URL"],
         mongodb_backend_settings=app.config["CELERY_MONGODB_BACKEND_SETTINGS"],
     )
-    celery.conf.update({"app_config": app.config})
-    celery.autodiscover_tasks(
+    created.conf.update({"app_config": app.config})
+    created.autodiscover_tasks(
         packages=[
             "app.tasks.email",
             "app.tasks.file_parse",
@@ -155,34 +143,17 @@ def create_celery():
         ],
         related_name=None,
     )
-    celery.conf.task_routes = {
+    created.conf.task_routes = {
         "tasks.ocr_task": {"queue": "ocr"},
         "tasks.output_project_task": {"queue": "output"},
         "tasks.import_from_labelplus_task": {"queue": "output"},
     }
-    return celery
+    return created
 
 
 celery = create_celery()
 
 
-def create_about_to_shutdown_flag():
-    try:
-        os.mkdir(CELERY_ABOUT_TO_SHUTDOWN_FLAG)
-    except Exception:
-        pass
-
-
-@worker_shutting_down.connect
-def when_shutdown(**kwargs):
-    create_about_to_shutdown_flag()
-
-
-def about_to_shutdown():
-    """检测 Celery 是否将要关闭"""
-    return os.path.isdir(CELERY_ABOUT_TO_SHUTDOWN_FLAG)
-
-
 @babel.localeselector
 def get_locale():
     current_user = g.get("current_user")

app/apis/team.py

@@ -339,6 +339,9 @@ def post(self, team: Team):
 
 
 class TeamProjectImportAPI(MoeAPIView):
+    """
+    Create new Project in current ProjectSet/Team, from exported Project JSON
+    """
     @token_required
     @fetch_model(Team)
     @fetch_model(ProjectSet)

app/tasks/import_from_labelplus.py

@@ -14,6 +14,7 @@
 from celery.utils.log import get_task_logger
 from app.utils.labelplus import load_from_labelplus
 from app.constants.source import SourcePositionType
+from celery.result import AsyncResult
 
 logger = get_task_logger(__name__)
 
@@ -99,7 +100,7 @@ def import_from_labelplus_task(project_id):
     return f"成功：Project {project_id}"
 
 
-def import_from_labelplus(project_id, /, *, run_sync=False):
+def import_from_labelplus(project_id, /, *, run_sync=False) -> SyncResult | AsyncResult:
     alive_workers = celery.control.ping()
     if len(alive_workers) == 0 or run_sync:
         # 同步执行

app/tasks/ocr.py

@@ -8,7 +8,7 @@
 from uuid import uuid4
 
 import requests
-from app import celery, gs_vision, oss, about_to_shutdown
+from app import celery, gs_vision, oss
 from app.constants.file import FileType, ImageOCRPercent, ParseErrorType, ParseStatus
 from app.models import connect_db
 from app.tasks import SyncResult
@@ -416,6 +416,8 @@ def inc(self, id, error_name, step=1):
 
 
 def merge_and_ocr(parsing_images, /, *, parse_alone=False):
+    if 2 > 1:
+        raise NotImplementedError("OCR under reconstruction")
     if not parsing_images:
         return
     oss_file_prefix = celery.conf.app_config["OSS_FILE_PREFIX"]
@@ -532,8 +534,6 @@ def merge_and_ocr(parsing_images, /, *, parse_alone=False):
                     parse_error_type=ParseErrorType.IMAGE_OCR_SERVER_DISCONNECT,
                 )
             continue
-        if about_to_shutdown() is True:
-            raise AboutToShutdownError
         for image_data in merged_images_data:
             image = image_data["image"]
             image.update(image_ocr_percent=ImageOCRPercent.LABELING)