Fix hanging upload of large files

bytes/bytes/api/router.py

@@ -1,10 +1,10 @@
 from uuid import UUID
 
 import structlog
-from asgiref.sync import async_to_sync
 from cachetools import TTLCache, cached
 from fastapi import APIRouter, Depends, HTTPException, Query, Request
 from fastapi.responses import Response
+from starlette.concurrency import run_in_threadpool
 from starlette.responses import JSONResponse
 
 from bytes.api.models import RawResponse
@@ -147,7 +147,7 @@ def get_normalizer_meta(
 
 
 @router.post("/raw", tags=[RAW_TAG])
-def create_raw(
+async def create_raw(
     request: Request,
     boefje_meta_id: UUID,
     mime_types: list[str] | None = Query(None),
@@ -162,13 +162,10 @@ def create_raw(
         if meta_repository.has_raw(meta, parsed_mime_types):
             return RawResponse(status="success", message="Raw data already present")
 
-        # FastAPI/starlette only has async versions of the Request methods, but
-        # all our code is sync, so we wrap it in async_to_sync.
-        data = async_to_sync(request.body)()
-
+        data = await request.body()
         raw_data = RawData(value=data, boefje_meta=meta, mime_types=parsed_mime_types)
         with meta_repository:
-            raw_id = meta_repository.save_raw(raw_data)
+            raw_id = await run_in_threadpool(meta_repository.save_raw, raw_data)
 
         event = RawFileReceived(
             organization=meta.organization,
@@ -178,7 +175,7 @@ def create_raw(
                 mime_types=raw_data.mime_types,
             ),
         )
-        event_manager.publish(event)
+        await run_in_threadpool(event_manager.publish, event)
     except Exception as error:
         logger.exception("Error saving raw data")
         raise HTTPException(status_code=500, detail="Could not save raw data") from error

bytes/pyproject.toml

@@ -18,7 +18,6 @@ pynacl = "^1.5.0"
 rfc3161ng = "^2.1.3"
 sqlalchemy = "^1.4.48"
 uvicorn = "^0.29.0"
-asgiref = "^3.8.1"
 
 # OpenTelemetry
 opentelemetry-sdk = "^1.26.0"

bytes/requirements-dev.txt

@@ -652,30 +652,51 @@ sniffio==1.3.1 ; python_version >= "3.10" and python_version < "4.0" \
     --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \
     --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc
 sqlalchemy==1.4.51 ; python_version >= "3.10" and python_version < "4.0" \
+    --hash=sha256:0525c4905b4b52d8ccc3c203c9d7ab2a80329ffa077d4bacf31aefda7604dc65 \
     --hash=sha256:0535d5b57d014d06ceeaeffd816bb3a6e2dddeb670222570b8c4953e2d2ea678 \
     --hash=sha256:0892e7ac8bc76da499ad3ee8de8da4d7905a3110b952e2a35a940dab1ffa550e \
     --hash=sha256:0d661cff58c91726c601cc0ee626bf167b20cc4d7941c93c5f3ac28dc34ddbea \
+    --hash=sha256:1980e6eb6c9be49ea8f89889989127daafc43f0b1b6843d71efab1514973cca0 \
+    --hash=sha256:1a09d5bd1a40d76ad90e5570530e082ddc000e1d92de495746f6257dc08f166b \
     --hash=sha256:245c67c88e63f1523e9216cad6ba3107dea2d3ee19adc359597a628afcabfbcb \
     --hash=sha256:2ad16880ccd971ac8e570550fbdef1385e094b022d6fc85ef3ce7df400dddad3 \
     --hash=sha256:2be4e6294c53f2ec8ea36486b56390e3bcaa052bf3a9a47005687ccf376745d1 \
     --hash=sha256:2c55040d8ea65414de7c47f1a23823cd9f3fad0dc93e6b6b728fee81230f817b \
     --hash=sha256:352df882088a55293f621328ec33b6ffca936ad7f23013b22520542e1ab6ad1b \
     --hash=sha256:3823dda635988e6744d4417e13f2e2b5fe76c4bf29dd67e95f98717e1b094cad \
     --hash=sha256:38ef80328e3fee2be0a1abe3fe9445d3a2e52a1282ba342d0dab6edf1fef4707 \
+    --hash=sha256:39b02b645632c5fe46b8dd30755682f629ffbb62ff317ecc14c998c21b2896ff \
+    --hash=sha256:3b0cd89a7bd03f57ae58263d0f828a072d1b440c8c2949f38f3b446148321171 \
     --hash=sha256:3ec7a0ed9b32afdf337172678a4a0e6419775ba4e649b66f49415615fa47efbd \
+    --hash=sha256:3f0ef620ecbab46e81035cf3dedfb412a7da35340500ba470f9ce43a1e6c423b \
+    --hash=sha256:50e074aea505f4427151c286955ea025f51752fa42f9939749336672e0674c81 \
     --hash=sha256:55e699466106d09f028ab78d3c2e1f621b5ef2c8694598242259e4515715da7c \
+    --hash=sha256:5e180fff133d21a800c4f050733d59340f40d42364fcb9d14f6a67764bdc48d2 \
     --hash=sha256:6cacc0b2dd7d22a918a9642fc89840a5d3cee18a0e1fe41080b1141b23b10916 \
+    --hash=sha256:7af40425ac535cbda129d9915edcaa002afe35d84609fd3b9d6a8c46732e02ee \
+    --hash=sha256:7d8139ca0b9f93890ab899da678816518af74312bb8cd71fb721436a93a93298 \
     --hash=sha256:7deeae5071930abb3669b5185abb6c33ddfd2398f87660fafdb9e6a5fb0f3f2f \
     --hash=sha256:86a22143a4001f53bf58027b044da1fb10d67b62a785fc1390b5c7f089d9838c \
     --hash=sha256:8ca484ca11c65e05639ffe80f20d45e6be81fbec7683d6c9a15cd421e6e8b340 \
+    --hash=sha256:8d1d7d63e5d2f4e92a39ae1e897a5d551720179bb8d1254883e7113d3826d43c \
+    --hash=sha256:8e702e7489f39375601c7ea5a0bef207256828a2bc5986c65cb15cd0cf097a87 \
+    --hash=sha256:a055ba17f4675aadcda3005df2e28a86feb731fdcc865e1f6b4f209ed1225cba \
+    --hash=sha256:a33cb3f095e7d776ec76e79d92d83117438b6153510770fcd57b9c96f9ef623d \
+    --hash=sha256:a61184c7289146c8cff06b6b41807c6994c6d437278e72cf00ff7fe1c7a263d1 \
     --hash=sha256:af55cc207865d641a57f7044e98b08b09220da3d1b13a46f26487cc2f898a072 \
+    --hash=sha256:b00cf0471888823b7a9f722c6c41eb6985cf34f077edcf62695ac4bed6ec01ee \
+    --hash=sha256:b03850c290c765b87102959ea53299dc9addf76ca08a06ea98383348ae205c99 \
     --hash=sha256:b97fd5bb6b7c1a64b7ac0632f7ce389b8ab362e7bd5f60654c2a418496be5d7f \
     --hash=sha256:c37bc677690fd33932182b85d37433845de612962ed080c3e4d92f758d1bd894 \
+    --hash=sha256:cecb66492440ae8592797dd705a0cbaa6abe0555f4fa6c5f40b078bd2740fc6b \
+    --hash=sha256:d0a83afab5e062abffcdcbcc74f9d3ba37b2385294dd0927ad65fc6ebe04e054 \
     --hash=sha256:d3cf56cc36d42908495760b223ca9c2c0f9f0002b4eddc994b24db5fcb86a9e4 \
     --hash=sha256:e646b19f47d655261b22df9976e572f588185279970efba3d45c377127d35349 \
     --hash=sha256:e7908c2025eb18394e32d65dd02d2e37e17d733cdbe7d78231c2b6d7eb20cdb9 \
     --hash=sha256:e8f2df79a46e130235bc5e1bbef4de0583fb19d481eaa0bffa76e8347ea45ec6 \
+    --hash=sha256:eaeeb2464019765bc4340214fca1143081d49972864773f3f1e95dba5c7edc7d \
     --hash=sha256:eb18549b770351b54e1ab5da37d22bc530b8bfe2ee31e22b9ebe650640d2ef12 \
+    --hash=sha256:f2e5b6f5cf7c18df66d082604a1d9c7a2d18f7d1dbe9514a2afaccbb51cc4fc3 \
     --hash=sha256:f8cafa6f885a0ff5e39efa9325195217bb47d5929ab0051636610d24aef45ade
 starlette==0.37.2 ; python_version >= "3.10" and python_version < "4.0" \
     --hash=sha256:6fe59f29268538e5d0d182f2791a479a0c64638e6935d1c6989e63fb2699c6ee \

bytes/requirements.txt

@@ -637,30 +637,51 @@ sniffio==1.3.1 ; python_version >= "3.10" and python_version < "4.0" \
     --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \
     --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc
 sqlalchemy==1.4.51 ; python_version >= "3.10" and python_version < "4.0" \
+    --hash=sha256:0525c4905b4b52d8ccc3c203c9d7ab2a80329ffa077d4bacf31aefda7604dc65 \
     --hash=sha256:0535d5b57d014d06ceeaeffd816bb3a6e2dddeb670222570b8c4953e2d2ea678 \
     --hash=sha256:0892e7ac8bc76da499ad3ee8de8da4d7905a3110b952e2a35a940dab1ffa550e \
     --hash=sha256:0d661cff58c91726c601cc0ee626bf167b20cc4d7941c93c5f3ac28dc34ddbea \
+    --hash=sha256:1980e6eb6c9be49ea8f89889989127daafc43f0b1b6843d71efab1514973cca0 \
+    --hash=sha256:1a09d5bd1a40d76ad90e5570530e082ddc000e1d92de495746f6257dc08f166b \
     --hash=sha256:245c67c88e63f1523e9216cad6ba3107dea2d3ee19adc359597a628afcabfbcb \
     --hash=sha256:2ad16880ccd971ac8e570550fbdef1385e094b022d6fc85ef3ce7df400dddad3 \
     --hash=sha256:2be4e6294c53f2ec8ea36486b56390e3bcaa052bf3a9a47005687ccf376745d1 \
     --hash=sha256:2c55040d8ea65414de7c47f1a23823cd9f3fad0dc93e6b6b728fee81230f817b \
     --hash=sha256:352df882088a55293f621328ec33b6ffca936ad7f23013b22520542e1ab6ad1b \
     --hash=sha256:3823dda635988e6744d4417e13f2e2b5fe76c4bf29dd67e95f98717e1b094cad \
     --hash=sha256:38ef80328e3fee2be0a1abe3fe9445d3a2e52a1282ba342d0dab6edf1fef4707 \
+    --hash=sha256:39b02b645632c5fe46b8dd30755682f629ffbb62ff317ecc14c998c21b2896ff \
+    --hash=sha256:3b0cd89a7bd03f57ae58263d0f828a072d1b440c8c2949f38f3b446148321171 \
     --hash=sha256:3ec7a0ed9b32afdf337172678a4a0e6419775ba4e649b66f49415615fa47efbd \
+    --hash=sha256:3f0ef620ecbab46e81035cf3dedfb412a7da35340500ba470f9ce43a1e6c423b \
+    --hash=sha256:50e074aea505f4427151c286955ea025f51752fa42f9939749336672e0674c81 \
     --hash=sha256:55e699466106d09f028ab78d3c2e1f621b5ef2c8694598242259e4515715da7c \
+    --hash=sha256:5e180fff133d21a800c4f050733d59340f40d42364fcb9d14f6a67764bdc48d2 \
     --hash=sha256:6cacc0b2dd7d22a918a9642fc89840a5d3cee18a0e1fe41080b1141b23b10916 \
+    --hash=sha256:7af40425ac535cbda129d9915edcaa002afe35d84609fd3b9d6a8c46732e02ee \
+    --hash=sha256:7d8139ca0b9f93890ab899da678816518af74312bb8cd71fb721436a93a93298 \
     --hash=sha256:7deeae5071930abb3669b5185abb6c33ddfd2398f87660fafdb9e6a5fb0f3f2f \
     --hash=sha256:86a22143a4001f53bf58027b044da1fb10d67b62a785fc1390b5c7f089d9838c \
     --hash=sha256:8ca484ca11c65e05639ffe80f20d45e6be81fbec7683d6c9a15cd421e6e8b340 \
+    --hash=sha256:8d1d7d63e5d2f4e92a39ae1e897a5d551720179bb8d1254883e7113d3826d43c \
+    --hash=sha256:8e702e7489f39375601c7ea5a0bef207256828a2bc5986c65cb15cd0cf097a87 \
+    --hash=sha256:a055ba17f4675aadcda3005df2e28a86feb731fdcc865e1f6b4f209ed1225cba \
+    --hash=sha256:a33cb3f095e7d776ec76e79d92d83117438b6153510770fcd57b9c96f9ef623d \
+    --hash=sha256:a61184c7289146c8cff06b6b41807c6994c6d437278e72cf00ff7fe1c7a263d1 \
     --hash=sha256:af55cc207865d641a57f7044e98b08b09220da3d1b13a46f26487cc2f898a072 \
+    --hash=sha256:b00cf0471888823b7a9f722c6c41eb6985cf34f077edcf62695ac4bed6ec01ee \
+    --hash=sha256:b03850c290c765b87102959ea53299dc9addf76ca08a06ea98383348ae205c99 \
     --hash=sha256:b97fd5bb6b7c1a64b7ac0632f7ce389b8ab362e7bd5f60654c2a418496be5d7f \
     --hash=sha256:c37bc677690fd33932182b85d37433845de612962ed080c3e4d92f758d1bd894 \
+    --hash=sha256:cecb66492440ae8592797dd705a0cbaa6abe0555f4fa6c5f40b078bd2740fc6b \
+    --hash=sha256:d0a83afab5e062abffcdcbcc74f9d3ba37b2385294dd0927ad65fc6ebe04e054 \
     --hash=sha256:d3cf56cc36d42908495760b223ca9c2c0f9f0002b4eddc994b24db5fcb86a9e4 \
     --hash=sha256:e646b19f47d655261b22df9976e572f588185279970efba3d45c377127d35349 \
     --hash=sha256:e7908c2025eb18394e32d65dd02d2e37e17d733cdbe7d78231c2b6d7eb20cdb9 \
     --hash=sha256:e8f2df79a46e130235bc5e1bbef4de0583fb19d481eaa0bffa76e8347ea45ec6 \
+    --hash=sha256:eaeeb2464019765bc4340214fca1143081d49972864773f3f1e95dba5c7edc7d \
     --hash=sha256:eb18549b770351b54e1ab5da37d22bc530b8bfe2ee31e22b9ebe650640d2ef12 \
+    --hash=sha256:f2e5b6f5cf7c18df66d082604a1d9c7a2d18f7d1dbe9514a2afaccbb51cc4fc3 \
     --hash=sha256:f8cafa6f885a0ff5e39efa9325195217bb47d5929ab0051636610d24aef45ade
 starlette==0.37.2 ; python_version >= "3.10" and python_version < "4.0" \
     --hash=sha256:6fe59f29268538e5d0d182f2791a479a0c64638e6935d1c6989e63fb2699c6ee \

bytes/tests/client.py

@@ -5,7 +5,7 @@
 from uuid import UUID
 
 import httpx
-from httpx import HTTPError
+from httpx import HTTPError, HTTPStatusError
 
 from bytes.models import BoefjeMeta, NormalizerMeta
 from bytes.repositories.meta_repository import BoefjeMetaFilter, NormalizerMetaFilter, RawDataFilter
@@ -21,7 +21,7 @@ def wrapper(self, *args, **kwargs):  # type: ignore
         try:
             return function(self, *args, **kwargs)
         except HTTPError as error:
-            if error.response.status_code != 401:
+            if not isinstance(error, HTTPStatusError) or error.response.status_code != 401:
                 raise
 
             self.login()