Update all project dependencies

.github/workflows/test-python.yaml

@@ -5,5 +5,5 @@ jobs:
   test-python:
     uses: winds-mobi/winds-mobi-cicd/.github/workflows/test-python.yaml@main
     with:
-      python_version: 3.11.4
-      poetry_version: 1.5.1
+      python_version: 3.11.9
+      poetry_version: 1.8.3

.tool-versions

@@ -1 +1 @@
-python 3.11.4
+python 3.11.9

Dockerfile

@@ -1,7 +1,4 @@
-FROM python:3.11.4-slim-bullseye AS base
-
-ENV LANG C.UTF-8
-ENV LC_ALL C.UTF-8
+FROM python:3.11.9-slim-bookworm AS base
 
 RUN apt update; \
     apt --yes --no-install-recommends install python3-scipy
@@ -10,7 +7,7 @@ FROM base AS python
 
 RUN apt update; \
     apt --yes --no-install-recommends install build-essential curl
-RUN curl -sSL https://install.python-poetry.org | python - --version 1.4.1
+RUN curl -sSL https://install.python-poetry.org | python - --version 1.8.3
 
 COPY . .
 RUN POETRY_VIRTUALENVS_IN_PROJECT=true /root/.local/bin/poetry install --without dev

docker-cmd.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 if [[ $TELEMETRY_DISABLED ]]; then
-  uvicorn --proxy-headers --root-path "$ROOT_PATH" --host 0.0.0.0 --port "$PORT" winds_mobi_api.main:app
+  uvicorn --log-config=winds_mobi_api/logging.yaml --proxy-headers --root-path "$ROOT_PATH" --host 0.0.0.0 --port "$PORT" winds_mobi_api.main:app
 else
-  opentelemetry-instrument --traces_exporter otlp uvicorn --proxy-headers --root-path "$ROOT_PATH" --host 0.0.0.0 --port "$PORT" winds_mobi_api.main:app
+  opentelemetry-instrument --traces_exporter otlp uvicorn --log-config=winds_mobi_api/logging.yaml --proxy-headers --root-path "$ROOT_PATH" --host 0.0.0.0 --port "$PORT" winds_mobi_api.main:app
 fi

pyproject.toml

@@ -9,32 +9,32 @@ packages = [
 ]
 
 [tool.poetry.dependencies]
-python = "3.11.*"
+python = "3.11.9"
 
-aiocache = "0.12.0"
-fastapi = "0.95.0"
-motor = "3.1.1"
-opentelemetry-distro = {extras = ["otlp"], version = "0.40b0"}
-opentelemetry-instrumentation = "0.40b0"
-opentelemetry-instrumentation-fastapi = "0.40b0"
-opentelemetry-instrumentation-pymongo = "0.40b0"
-orjson = "3.8.8"
+aiocache = "0.12.2"
+fastapi = "0.111.0"
+motor = "3.5.1"
+opentelemetry-distro = {extras = ["otlp"], version = "0.46b0"}
+opentelemetry-instrumentation = "0.46b0"
+opentelemetry-instrumentation-fastapi = "0.46b0"
+opentelemetry-instrumentation-pymongo = "0.46b0"
+orjson = "3.10.6"
 parse-accept-language = "0.1.2"
-pyaml = "21.10.1"
-pydantic = "1.10.7"
-scipy = "1.10.1"
+pyaml = "24.4.0"
+pydantic = "2.8.2"
+pydantic-settings = "2.3.4"
+scipy = "1.14.0"
 sentry-asgi = "0.2.0"
-sentry-sdk = "1.28.1"
+sentry-sdk = "2.9.0"
 stop-words = "2018.7.23"
-ujson = "5.7.0" # Optional dependency used by aiocache
-uvicorn = {extras = ["standard"], version = "0.21.1"}
+ujson = "5.10.0" # Optional dependency used by aiocache
 
 [tool.poetry.group.dev.dependencies]
-black = "22.10.0"
-flake8 = "6.0.0"
-isort = "5.10.1"
+black = "24.4.2"
+flake8 = "7.1.0"
+isort = "5.13.2"
 locust = "*"
-pytest = "7.2.0"
+pytest = "8.2.2"
 python-dotenv = "*"
 
 [build-system]

winds_mobi_api/logging.yaml

@@ -1,5 +1,5 @@
 version: 1
-disable_existing_loggers: True
+disable_existing_loggers: False
 formatters:
   console:
     format: "%(asctime)s %(levelname)s [%(name)s] | %(message)s"
@@ -12,6 +12,10 @@ handlers:
 loggers:
   uvicorn.error:
     level: INFO
+  uvicorn.access:
+    level: WARNING
+  winds_mobi_api:
+    level: INFO
 root:
   handlers: [ console ]
   level: INFO

winds_mobi_api/main.py

@@ -1,8 +1,10 @@
 import asyncio
 import logging
+from contextlib import asynccontextmanager
 from logging.config import dictConfig
 from pathlib import Path
 
+import bson
 import pymongo
 import sentry_sdk
 import uvloop
@@ -27,9 +29,20 @@
 
 asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
 
+
+@asynccontextmanager
+async def lifespan(fastapi: FastAPI):
+    database._mongodb = motor_asyncio.AsyncIOMotorClient(settings.mongodb_url).get_database()
+    database._mongodb_sync = MongoClient(settings.mongodb_url).get_database()
+    yield
+    database.mongodb().client.close()
+    database.mongodb_sync().client.close()
+
+
 app = FastAPI(
     title="winds.mobi",
     version="2.3",
+    lifespan=lifespan,
     openapi_prefix=settings.openapi_prefix,
     docs_url=f"/{settings.doc_path}",
     description="""### Feel free to "fair use" this API
@@ -60,16 +73,17 @@
 app.add_middleware(SentryMiddleware)
 
 
-@app.on_event("startup")
-async def startup_event():
-    database._mongodb = motor_asyncio.AsyncIOMotorClient(settings.mongodb_url).get_database()
-    database._mongodb_sync = MongoClient(settings.mongodb_url).get_database()
+@app.exception_handler(pymongo.errors.OperationFailure)
+async def mongodb_client_exception(request, e):
+    log.warning(f"Mongodb failure: {e.details['errmsg']}")
+    return JSONResponse({"detail": e.details["errmsg"]}, status_code=400)
 
 
-@app.exception_handler(pymongo.errors.OperationFailure)
-async def mongo_exception(request, exc):
-    log.error("Mongodb error", exc_info=exc)
-    return JSONResponse({"detail": "Mongodb error"}, status_code=400)
+@app.exception_handler(pymongo.errors.PyMongoError)
+@app.exception_handler(bson.errors.BSONError)
+async def mongodb_server_exception(request, e):
+    log.error("Mongodb error", exc_info=e)
+    return JSONResponse({"detail": "Mongodb error"}, status_code=500)
 
 
 @app.get("/", include_in_schema=False)

winds_mobi_api/settings.py

@@ -1,10 +1,12 @@
-from pydantic import BaseSettings
+from typing import Optional
+
+from pydantic_settings import BaseSettings
 
 
 class Settings(BaseSettings):
     port: int = 8000
     environment: str = "local"
-    sentry_dsn: str = None
+    sentry_dsn: Optional[str] = None
     mongodb_url: str
     openapi_prefix: str = ""
     doc_path: str = "doc"

winds_mobi_api/views.py

@@ -1,4 +1,3 @@
-import asyncio
 import logging
 from datetime import datetime
 from typing import List, Union
@@ -8,6 +7,7 @@
 from fastapi import APIRouter, Header, HTTPException, Path, Query
 from fastapi.responses import ORJSONResponse
 from scipy import optimize
+from starlette.concurrency import run_in_threadpool
 from stop_words import StopWordError, get_stop_words
 
 from winds_mobi_api import database, diacritics
@@ -33,6 +33,11 @@ async def get_collection_names(**cache_kwargs):
     return await database.mongodb().list_collection_names()
 
 
+@cached(ttl=10 * 60)
+async def get_save_clusters():
+    return await database.mongodb().stations_clusters.find_one("save_clusters")
+
+
 def response(data):
     if settings.response_schema_validation:
         return data
@@ -129,7 +134,7 @@ async def find_stations(
         "Can be a duration in seconds or a absolute datetime, for example: 2019-08-16 15:30",
     ),
     is_highest_duplicates_rating: bool = Query(
-        False,
+        None,
         alias="is-highest-duplicates-rating",
         description="Return only stations with the highest duplicates rating (filter stations at the same place)",
     ),
@@ -183,7 +188,7 @@ async def find_stations(
     if last_measure is not None:
         timestamp = None
         if isinstance(last_measure, int):
-            timestamp = datetime.now().timestamp() - last_measure
+            timestamp = now - last_measure
         elif isinstance(last_measure, datetime):
             timestamp = last_measure.timestamp()
         if timestamp:
@@ -226,35 +231,31 @@ async def find_stations(
             }
         }
 
-        now = datetime.now().timestamp()
-        nb_stations = await database.mongodb().stations.count_documents(
-            {"status": {"$ne": "hidden"}, "last._id": {"$gt": now - 30 * 24 * 3600}}
-        )
+        def get_cluster_query(cluster: int):
+            return {**query, "clusters": {"$elemMatch": {"$lte": cluster}}}
 
-        def get_cluster_query(cluster):
-            return {**query, "clusters": {"$elemMatch": {"$lte": int(cluster)}}}
-
-        def count(x):
-            y = len(list(database.mongodb_sync().stations.find(get_cluster_query(x), {"_id": 1})))
-            return y - limit
-
-        def no_cluster_task():
-            return optimize.brentq(count, 1, nb_stations, maxiter=2, disp=False)
+        def no_cluster_task(min, max) -> float:
+            return optimize.brentq(
+                lambda x: database.mongodb_sync().stations.count_documents((get_cluster_query(int(x)))) - limit,
+                min,
+                max,
+                maxiter=2,
+                disp=False,
+            )
 
+        save_cluster = await get_save_clusters()
         try:
             # CPU and IO bound task using a sync library: executing it in a separated thread to not block the event loop
-            no_cluster = await asyncio.get_running_loop().run_in_executor(None, no_cluster_task)
+            no_cluster = int(await run_in_threadpool(no_cluster_task, min=save_cluster["min"], max=save_cluster["max"]))
         except ValueError:
             no_cluster = None
 
         if no_cluster:
             cursor = database.mongodb().stations.find(get_cluster_query(no_cluster), projection_dict)
-            stations = await cursor.to_list(None)
-            log.debug(f"limit={limit}, no_cluster={no_cluster:.0f} => {len(stations)}")
         else:
             cursor = database.mongodb().stations.find(query, projection_dict)
-            stations = await cursor.to_list(None)
-            log.debug(f"limit={limit} => {len(stations)}")
+        stations = await cursor.to_list(None)
+        log.debug(f"no_cluster={no_cluster}, limit={limit} => {len(stations)}")
         return response(stations)
 
     if ids: