Merge branch 'master' into enh/user-notifications

.github/workflows/ci-testing-deploy.yml

@@ -450,6 +450,51 @@ jobs:
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
 
+  unit-test-webserver-04:
+      needs: changes
+      if: ${{ needs.changes.outputs.webserver == 'true' || github.event_name == 'push' }}
+      timeout-minutes: 25 # if this timeout gets too small, then split the tests
+      name: "[unit] webserver 04"
+      runs-on: ${{ matrix.os }}
+      strategy:
+        matrix:
+          python: ["3.11"]
+          os: [ubuntu-22.04]
+        fail-fast: false
+      steps:
+        - uses: actions/checkout@v4
+        - name: setup docker buildx
+          id: buildx
+          uses: docker/setup-buildx-action@v3
+          with:
+            driver: docker-container
+        - name: setup python environment
+          uses: actions/setup-python@v5
+          with:
+            python-version: ${{ matrix.python }}
+        - name: install uv
+          uses: astral-sh/setup-uv@v3
+          with:
+            version: "0.4.x"
+            enable-cache: false
+            cache-dependency-glob: "**/web/server/requirements/ci.txt"
+        - name: show system version
+          run: ./ci/helpers/show_system_versions.bash
+        - name: install webserver
+          run: ./ci/github/unit-testing/webserver.bash install
+        - name: test
+          run: ./ci/github/unit-testing/webserver.bash test_with_db 04
+        - uses: codecov/[email protected]
+          env:
+            CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+          with:
+            flags: unittests #optional
+        - name: Upload test results to Codecov
+          if: ${{ !cancelled() }}
+          uses: codecov/test-results-action@v1
+          with:
+            token: ${{ secrets.CODECOV_TOKEN }}
+
   unit-test-storage:
     needs: changes
     if: ${{ needs.changes.outputs.storage == 'true' || github.event_name == 'push' }}
@@ -1875,6 +1920,7 @@ jobs:
         unit-test-webserver-01,
         unit-test-webserver-02,
         unit-test-webserver-03,
+        unit-test-webserver-04,
       ]
     runs-on: ubuntu-latest
     steps:

services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py

@@ -135,7 +135,7 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings):
         description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!",
     )
     DY_SIDECAR_LOG_FILTER_MAPPING: dict[LoggerName, list[MessageSubstring]] = Field(
-        default={},
+        default_factory=dict,
         env=["DY_SIDECAR_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"],
         description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.",
     )

services/resource-usage-tracker/docker/boot.sh

@@ -56,7 +56,7 @@ if [ "${SC_BOOT_MODE}" = "debug" ]; then
       --log-level \"${SERVER_LOG_LEVEL}\"
   "
 else
-  exec uvicorn simcore_service_resource_usage_tracker.web_main:the_app \
+  exec uvicorn simcore_service_resource_usage_tracker.main:the_app \
     --host 0.0.0.0 \
     --log-level "${SERVER_LOG_LEVEL}"
 fi

services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rest/_health.py

@@ -7,7 +7,7 @@
 from models_library.errors import RABBITMQ_CLIENT_UNHEALTHY_MSG
 from servicelib.rabbitmq import RabbitMQClient
 
-from ...modules.rabbitmq import get_rabbitmq_client_from_request
+from ...services.modules.rabbitmq import get_rabbitmq_client_from_request
 
 logger = logging.getLogger(__name__)
 

services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rest/_resource_tracker.py

@@ -12,11 +12,7 @@
 )
 from models_library.resource_tracker import CreditTransactionId
 
-from ...services import (
-    resource_tracker_credit_transactions,
-    resource_tracker_pricing_plans,
-    resource_tracker_pricing_units,
-)
+from ...services import credit_transactions, pricing_plans, pricing_units
 
 _logger = logging.getLogger(__name__)
 
@@ -38,9 +34,7 @@
 async def get_credit_transactions_sum(
     wallet_total_credits: Annotated[
         WalletTotalCredits,
-        Depends(
-            resource_tracker_credit_transactions.sum_credit_transactions_by_product_and_wallet
-        ),
+        Depends(credit_transactions.sum_credit_transactions_by_product_and_wallet),
     ],
 ):
     return wallet_total_credits
@@ -56,7 +50,7 @@ async def get_credit_transactions_sum(
 async def create_credit_transaction(
     transaction_id: Annotated[
         CreditTransactionId,
-        Depends(resource_tracker_credit_transactions.create_credit_transaction),
+        Depends(credit_transactions.create_credit_transaction),
     ],
 ):
     return {"credit_transaction_id": transaction_id}
@@ -77,7 +71,7 @@ async def create_credit_transaction(
 async def get_service_default_pricing_plan(
     service_pricing_plans: Annotated[
         PricingPlanGet,
-        Depends(resource_tracker_pricing_plans.get_service_default_pricing_plan),
+        Depends(pricing_plans.get_service_default_pricing_plan),
     ],
 ):
     return service_pricing_plans
@@ -93,7 +87,7 @@ async def get_service_default_pricing_plan(
 async def get_pricing_plan_unit(
     pricing_unit: Annotated[
         PricingUnitGet,
-        Depends(resource_tracker_pricing_units.get_pricing_unit),
+        Depends(pricing_units.get_pricing_unit),
     ]
 ):
     return pricing_unit

services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rest/dependencies.py

@@ -12,7 +12,7 @@
 from servicelib.fastapi.dependencies import get_app, get_reverse_url_mapper
 from sqlalchemy.ext.asyncio import AsyncEngine
 
-from ...modules.db.repositories._base import BaseRepository
+from ...services.modules.db.repositories._base import BaseRepository
 
 logger = logging.getLogger(__name__)
 

services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rpc/_resource_tracker.py

@@ -26,24 +26,21 @@
 from models_library.wallets import WalletID
 from pydantic import AnyUrl
 from servicelib.rabbitmq import RPCRouter
-from servicelib.rabbitmq.rpc_interfaces.resource_usage_tracker.errors import (
-    CustomResourceUsageTrackerError,
-)
 
 from ...core.settings import ApplicationSettings
-from ...modules.db.repositories.resource_tracker import ResourceTrackerRepository
-from ...modules.s3 import get_s3_client
-from ...services import resource_tracker_pricing_plans as pricing_plans
-from ...services import resource_tracker_pricing_units as pricing_units
-from ...services import resource_tracker_service_runs as service_runs
+from ...services import pricing_plans, pricing_units, service_runs
+from ...services.modules.db.repositories.resource_tracker import (
+    ResourceTrackerRepository,
+)
+from ...services.modules.s3 import get_s3_client
 
 router = RPCRouter()
 
 
 ## Service runs
 
 
-@router.expose(reraise_if_error_type=(CustomResourceUsageTrackerError,))
+@router.expose(reraise_if_error_type=())
 async def get_service_run_page(
     app: FastAPI,
     *,
@@ -69,7 +66,7 @@ async def get_service_run_page(
     )
 
 
-@router.expose(reraise_if_error_type=(CustomResourceUsageTrackerError,))
+@router.expose(reraise_if_error_type=())
 async def export_service_runs(
     app: FastAPI,
     *,
@@ -98,7 +95,7 @@ async def export_service_runs(
     )
 
 
-@router.expose(reraise_if_error_type=(CustomResourceUsageTrackerError,))
+@router.expose(reraise_if_error_type=())
 async def get_osparc_credits_aggregated_usages_page(
     app: FastAPI,
     *,
@@ -127,7 +124,7 @@ async def get_osparc_credits_aggregated_usages_page(
 ## Pricing plans
 
 
-@router.expose(reraise_if_error_type=(CustomResourceUsageTrackerError,))
+@router.expose(reraise_if_error_type=())
 async def get_pricing_plan(
     app: FastAPI,
     *,
@@ -141,7 +138,7 @@ async def get_pricing_plan(
     )
 
 
-@router.expose(reraise_if_error_type=(CustomResourceUsageTrackerError,))
+@router.expose(reraise_if_error_type=())
 async def list_pricing_plans(
     app: FastAPI,
     *,
@@ -153,7 +150,7 @@ async def list_pricing_plans(
     )
 
 
-@router.expose(reraise_if_error_type=(CustomResourceUsageTrackerError,))
+@router.expose(reraise_if_error_type=())
 async def create_pricing_plan(
     app: FastAPI,
     *,
@@ -165,7 +162,7 @@ async def create_pricing_plan(
     )
 
 
-@router.expose(reraise_if_error_type=(CustomResourceUsageTrackerError,))
+@router.expose(reraise_if_error_type=())
 async def update_pricing_plan(
     app: FastAPI,
     *,
@@ -182,7 +179,7 @@ async def update_pricing_plan(
 ## Pricing units
 
 
-@router.expose(reraise_if_error_type=(CustomResourceUsageTrackerError,))
+@router.expose(reraise_if_error_type=())
 async def get_pricing_unit(
     app: FastAPI,
     *,
@@ -198,7 +195,7 @@ async def get_pricing_unit(
     )
 
 
-@router.expose(reraise_if_error_type=(CustomResourceUsageTrackerError,))
+@router.expose(reraise_if_error_type=())
 async def create_pricing_unit(
     app: FastAPI,
     *,
@@ -212,7 +209,7 @@ async def create_pricing_unit(
     )
 
 
-@router.expose(reraise_if_error_type=(CustomResourceUsageTrackerError,))
+@router.expose(reraise_if_error_type=())
 async def update_pricing_unit(
     app: FastAPI,
     *,
@@ -229,7 +226,7 @@ async def update_pricing_unit(
 ## Pricing plan to service
 
 
-@router.expose(reraise_if_error_type=(CustomResourceUsageTrackerError,))
+@router.expose(reraise_if_error_type=())
 async def list_connected_services_to_pricing_plan_by_pricing_plan(
     app: FastAPI,
     *,
@@ -246,7 +243,7 @@ async def list_connected_services_to_pricing_plan_by_pricing_plan(
     return output
 
 
-@router.expose(reraise_if_error_type=(CustomResourceUsageTrackerError,))
+@router.expose(reraise_if_error_type=())
 async def connect_service_to_pricing_plan(
     app: FastAPI,
     *,

services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rpc/routes.py

@@ -4,7 +4,7 @@
 )
 from servicelib.rabbitmq import RPCRouter
 
-from ...modules.rabbitmq import get_rabbitmq_rpc_server
+from ...services.modules.rabbitmq import get_rabbitmq_rpc_server
 from . import _resource_tracker
 
 ROUTERS: list[RPCRouter] = [

services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/application.py

@@ -2,13 +2,7 @@
 
 from fastapi import FastAPI
 from servicelib.fastapi.openapi import override_fastapi_openapi_method
-from servicelib.fastapi.prometheus_instrumentation import (
-    setup_prometheus_instrumentation,
-)
 from servicelib.fastapi.tracing import setup_tracing
-from servicelib.rabbitmq.rpc_interfaces.resource_usage_tracker.errors import (
-    CustomResourceUsageTrackerError,
-)
 
 from .._meta import (
     API_VERSION,
@@ -20,12 +14,17 @@
 )
 from ..api.rest.routes import setup_api_routes
 from ..api.rpc.routes import setup_rpc_api_routes
-from ..modules.db import setup as setup_db
-from ..modules.rabbitmq import setup as setup_rabbitmq
-from ..modules.redis import setup as setup_redis
-from ..modules.s3 import setup as setup_s3
-from ..resource_tracker import setup as setup_resource_tracker
-from .errors import http404_error_handler
+from ..exceptions.handlers import setup_exception_handlers
+from ..services.background_task_periodic_heartbeat_check_setup import (
+    setup as setup_background_task_periodic_heartbeat_check,
+)
+from ..services.modules.db import setup as setup_db
+from ..services.modules.rabbitmq import setup as setup_rabbitmq
+from ..services.modules.redis import setup as setup_redis
+from ..services.modules.s3 import setup as setup_s3
+from ..services.process_message_running_service_setup import (
+    setup as setup_process_message_running_service,
+)
 from .settings import ApplicationSettings
 
 _logger = logging.getLogger(__name__)
@@ -52,18 +51,6 @@ def create_app(settings: ApplicationSettings) -> FastAPI:
     # PLUGINS SETUP
     setup_api_routes(app)
 
-    if app.state.settings.RESOURCE_USAGE_TRACKER_PROMETHEUS_INSTRUMENTATION_ENABLED:
-        setup_prometheus_instrumentation(app)
-    if app.state.settings.RESOURCE_USAGE_TRACKER_TRACING:
-        setup_tracing(
-            app,
-            app.state.settings.RESOURCE_USAGE_TRACKER_TRACING,
-            app.state.settings.APP_NAME,
-        )
-
-    # ERROR HANDLERS
-    app.add_exception_handler(CustomResourceUsageTrackerError, http404_error_handler)
-
     if settings.RESOURCE_USAGE_TRACKER_POSTGRES:
         setup_db(app)
     setup_redis(app)
@@ -72,8 +59,20 @@ def create_app(settings: ApplicationSettings) -> FastAPI:
         # Needed for CSV export functionality
         setup_s3(app)
 
-    setup_resource_tracker(app)
-    setup_rpc_api_routes(app)
+    setup_rpc_api_routes(app)  # Requires Rabbit, S3
+    setup_background_task_periodic_heartbeat_check(app)  # Requires Redis, DB
+
+    setup_process_message_running_service(app)  # Requires Rabbit
+
+    if app.state.settings.RESOURCE_USAGE_TRACKER_TRACING:
+        setup_tracing(
+            app,
+            app.state.settings.RESOURCE_USAGE_TRACKER_TRACING,
+            app.state.settings.APP_NAME,
+        )
+
+    # ERROR HANDLERS
+    setup_exception_handlers(app)
 
     # EVENTS
     async def _on_startup() -> None: