More server i18n (#30)

* disable custom roles API route

* create system roles with chinese id

* minor changes

* make ruff  happy

* disable doomed test

* fix test

* revise translations

* wip

* fix

* fix

* add BABEL_DEFAULT_LOCALE option

* attach Flask app context to Celery tasks

* tune log levels

* rename

* wrap flask_babel.gettext for debugging

* update translations

* minor fixes

* make ruff happy

.env.sample

@@ -13,6 +13,9 @@ SECRET_KEY=CHANGE_ME
 ADMIN_EMAIL=change_me@change_me.com
 ADMIN_INITIAL_PASSWORD=change_me
 
+# i18n locale for server-context
+BABEL_DEFAULT_LOCALE=en
+
 # mongodb database
 MONGODB_URI='mongodb://moeflow:PLEASE_CHANGE_THIS@moeflow-mongodb:27017/moeflow?authSource=admin'
 

app/apis/me.py

@@ -313,7 +313,9 @@ def get(self):
         p = MoePagination()
         teams = self.current_user.teams(skip=p.skip, limit=p.limit, word=word)
         # 获取团队用户关系
-        relations = TeamUserRelation.objects(group__in=teams, user=self.current_user)
+        relations: list[TeamUserRelation] = TeamUserRelation.objects(
+            group__in=teams, user=self.current_user
+        )
         # 构建字典用于快速匹配
         team_roles_data = {}
         for relation in relations:

app/config.py

@@ -36,7 +36,7 @@
 # -----------
 # i18n
 # -----------
-BABEL_DEFAULT_LOCALE = "zh"
+BABEL_DEFAULT_LOCALE = env.get("BABEL_DEFAULT_LOCALE", "zh")
 BABEL_DEFAULT_TIMEZONE = "UTC"
 # -----------
 # 其他设置

app/core/rbac.py

@@ -7,7 +7,7 @@
 import logging
 from app.exceptions import UserNotExistError, CreatorCanNotLeaveError
 
-from flask_babel import gettext, lazy_gettext
+from app.translations import gettext, lazy_gettext
 from mongoengine import (
     BooleanField,
     DateTimeField,

app/factory.py

@@ -1,5 +1,5 @@
 import logging
-from celery import Celery
+import celery
 from flask import Flask
 from flask_apikit import APIKit
 from flask_babel import Babel
@@ -8,7 +8,7 @@
 import app.config as _app_config
 from app.services.oss import OSS
 from .apis import register_apis
-from app.translations import get_locale
+from app.translations import get_request_locale
 
 from app.models import connect_db
 
@@ -43,22 +43,33 @@ def create_flask_app(app: Flask) -> Flask:
 
 def init_flask_app(app: Flask):
     register_apis(app)
-    babel.init_app(app, locale_selector=get_locale)
+    babel.init_app(
+        app,
+        locale_selector=get_request_locale,
+        default_locale=app_config["BABEL_DEFAULT_LOCALE"],
+    )
     apikit.init_app(app)
     logger.info(f"----- build id: {app_config['BUILD_ID']}")
     with app.app_context():
         logger.debug(
-            "站点支持语言: " + str([str(i) for i in babel.list_translations()])
+            "Server locale translations: "
+            + str([str(i) for i in babel.list_translations()])
         )
     oss.init(app.config)  # 文件储存
 
 
-def create_celery(app: Flask) -> Celery:
-    # 通过app配置创建celery实例
-    created = Celery(
+def create_celery(app: Flask) -> celery.Celery:
+    # see https://flask.palletsprojects.com/en/stable/patterns/celery/
+    class FlaskTask(celery.Task):
+        def __call__(self, *args: object, **kwargs: object) -> object:
+            with app.app_context():
+                return self.run(*args, **kwargs)
+
+    created = celery.Celery(
         app.name,
         broker=app.config["CELERY_BROKER_URL"],
         backend=app.config["CELERY_BACKEND_URL"],
+        task_cls=FlaskTask,
         **app.config["CELERY_BACKEND_SETTINGS"],
     )
     created.conf.update({"app_config": app.config})

app/models/__init__.py

@@ -6,7 +6,7 @@
 from mongoengine import connect
 
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.DEBUG)
+logger.setLevel(logging.INFO)
 
 
 def connect_db(config):

app/models/file.py

@@ -1221,8 +1221,8 @@ class Source(Document):
     meta = {"indexes": ["file", "blank", "rank"]}
 
     @classmethod
-    def by_id(cls, id) -> "Source":
-        source = cls.objects(id=id).first()
+    def by_id(cls, id_: str | ObjectId) -> "Source":
+        source = cls.objects(id=id_).first()
         if source is None:
             raise SourceNotExistError
         return source

app/models/project.py

@@ -310,11 +310,11 @@ def clear(self):
         self.delete()
 
     @classmethod
-    def by_id(cls, id):
-        set = cls.objects(id=id).first()
-        if set is None:
+    def by_id(cls, id: ObjectId):
+        project_set = cls.objects(id=id).first()
+        if project_set is None:
             raise ProjectSetNotExistError()
-        return set
+        return project_set
 
     def to_api(self):
         """

app/models/team.py

@@ -3,7 +3,7 @@
 from typing import List
 
 from flask import current_app
-from flask_babel import lazy_gettext, gettext
+from app.translations import lazy_gettext, gettext
 from mongoengine import (
     CASCADE,
     DENY,
@@ -470,7 +470,7 @@ def to_api(self, user=None):
         # 如果给了 role 则获取用户相关信息（角色等）
         role = None
         if user:
-            role = user.get_role(self)
+            role: TeamRole | None = user.get_role(self)
             if role:
                 role = role.to_api()
         return {

app/tasks/import_from_labelplus.py

@@ -56,35 +56,32 @@ def import_from_labelplus_task(project_id):
         )
         return f"失败：创建者不存在，Project {project_id}"
     try:
-        with app.app_context():
-            if target and creator:
+        if target and creator:
+            project.update(
+                import_from_labelplus_percent=0,
+                import_from_labelplus_status=ImportFromLabelplusStatus.RUNNING,
+            )
+            labelplus_data = load_from_labelplus(project.import_from_labelplus_txt)
+            file_count = len(labelplus_data)
+            for file_index, labelplus_file in enumerate(labelplus_data):
+                file = project.create_file(labelplus_file["file_name"])
+                for labelplus_label in labelplus_file["labels"]:
+                    source = file.create_source(
+                        content="",
+                        x=labelplus_label["x"],
+                        y=labelplus_label["y"],
+                        position_type=SourcePositionType.IN
+                        if labelplus_label["position_type"] == SourcePositionType.IN
+                        else SourcePositionType.OUT,
+                    )
+                    source.create_translation(
+                        content=labelplus_label["translation"],
+                        target=target,
+                        user=creator,
+                    )
                 project.update(
-                    import_from_labelplus_percent=0,
-                    import_from_labelplus_status=ImportFromLabelplusStatus.RUNNING,
+                    import_from_labelplus_percent=int((file_index / file_count) * 100)
                 )
-                labelplus_data = load_from_labelplus(project.import_from_labelplus_txt)
-                file_count = len(labelplus_data)
-                for file_index, labelplus_file in enumerate(labelplus_data):
-                    file = project.create_file(labelplus_file["file_name"])
-                    for labelplus_label in labelplus_file["labels"]:
-                        source = file.create_source(
-                            content="",
-                            x=labelplus_label["x"],
-                            y=labelplus_label["y"],
-                            position_type=SourcePositionType.IN
-                            if labelplus_label["position_type"] == SourcePositionType.IN
-                            else SourcePositionType.OUT,
-                        )
-                        source.create_translation(
-                            content=labelplus_label["translation"],
-                            target=target,
-                            user=creator,
-                        )
-                    project.update(
-                        import_from_labelplus_percent=int(
-                            (file_index / file_count) * 100
-                        )
-                    )
     except Exception:
         logger.exception(Exception)
         project.update(

app/tasks/output_team_projects.py

@@ -37,32 +37,31 @@ def output_team_projects_task(team_id, current_user_id):
     app = Flask(__name__)
     app.config.from_object(celery.conf.app_config)
 
-    with app.app_context():
-        OUTPUT_WAIT_SECONDS = celery.conf.app_config.get("OUTPUT_WAIT_SECONDS", 60 * 5)
-        current_user = User.by_id(current_user_id)
+    OUTPUT_WAIT_SECONDS = celery.conf.app_config.get("OUTPUT_WAIT_SECONDS", 60 * 5)
+    current_user = User.by_id(current_user_id)
 
-        team = Team.by_id(team_id)
-        for project in team.projects(status=ProjectStatus.WORKING):
-            for target in project.targets():
-                # 等待一定时间后允许再次导出
-                last_output = target.outputs().first()
-                if last_output and (
-                    datetime.datetime.utcnow() - last_output.create_time
-                    < datetime.timedelta(seconds=OUTPUT_WAIT_SECONDS)
-                ):
-                    continue
-                # 删除三个导出之前的
-                old_targets = target.outputs().skip(2)
-                Output.delete_real_files(old_targets)
-                old_targets.delete()
-                # 创建新target
-                output = Output.create(
-                    project=project,
-                    target=target,
-                    user=current_user,
-                    type=OutputTypes.ALL,
-                )
-                output_project(str(output.id))
+    team = Team.by_id(team_id)
+    for project in team.projects(status=ProjectStatus.WORKING):
+        for target in project.targets():
+            # 等待一定时间后允许再次导出
+            last_output = target.outputs().first()
+            if last_output and (
+                datetime.datetime.utcnow() - last_output.create_time
+                < datetime.timedelta(seconds=OUTPUT_WAIT_SECONDS)
+            ):
+                continue
+            # 删除三个导出之前的
+            old_targets = target.outputs().skip(2)
+            Output.delete_real_files(old_targets)
+            old_targets.delete()
+            # 创建新target
+            output = Output.create(
+                project=project,
+                target=target,
+                user=current_user,
+                type=OutputTypes.ALL,
+            )
+            output_project(str(output.id))
 
     return f"成功：已创建 Team <{str(team.id)}> 所有项目的导出任务"
 

app/translations/__init__.py

@@ -2,26 +2,30 @@
 from typing import Optional
 from flask import g, request
 from app.constants.locale import Locale
+import flask_babel
 
 logger = logging.getLogger(__name__)
-# logger.setLevel(logging.DEBUG)
+logger.setLevel(logging.INFO)
 
 
-def get_locale() -> Optional[str]:
+def get_request_locale() -> Optional[str]:
     current_user = g.get("current_user")
+    req_header = f"{request.method} {request.path}"
     if (
         current_user
         and current_user.locale
         and current_user.locale != "auto"
         and current_user.locale in Locale.ids()
     ):
-        # NOTE User.locale is not used
-        logging.debug("locale from user %s", current_user.locale)
+        # NOTE User.locale is not used , so this won't get called
+        logging.debug(
+            "%s set locale=%s from user pref", req_header, current_user.locale
+        )
         return current_user.locale
     # "zh" locale asssets is created from hardcoded strings
     # "en" locale are machine translated
     best_match = request.accept_languages.best_match(["zh", "en"], default="en")
-    logging.debug("locale from req %s", best_match)
+    logging.debug("%s set locale=%s from req", req_header, best_match)
     return best_match
 
 
@@ -32,3 +36,17 @@ def get_locale() -> Optional[str]:
 #     if current_user:
 #         if current_user.timezone:
 #             return current_user.timezone
+
+
+def gettext(msgid: str):
+    translated = flask_babel.gettext(msgid)
+    logger.debug(
+        f"get_text({msgid}, locale={flask_babel.get_locale()}) -> {translated}"
+    )
+    return translated
+
+
+def lazy_gettext(msgid: str):
+    translated = flask_babel.LazyString(lambda: gettext(msgid))
+    # logger.debug(f"lazy_get_text({msgid}) -> {translated}")
+    return translated