Fix hanging upload of large files (#3489)

Co-authored-by: Jeroen Dekkers <[email protected]> Co-authored-by: Jan Klopper <[email protected]>
minvws · Sep 11, 2024 · 68eef2c · 68eef2c
1 parent de06d4a
commit 68eef2c
Show file tree

Hide file tree

Showing 6 changed files with 83 additions and 4 deletions.
diff --git a/bytes/poetry.lock b/bytes/poetry.lock
diff --git a/bytes/pyproject.toml b/bytes/pyproject.toml
@@ -18,7 +18,6 @@ pynacl = "^1.5.0"
 rfc3161ng = "^2.1.3"
 sqlalchemy = "^1.4.48"
 uvicorn = "^0.29.0"
-asgiref = "^3.8.1"
 
 # OpenTelemetry
 opentelemetry-sdk = "^1.26.0"

diff --git a/bytes/requirements-dev.txt b/bytes/requirements-dev.txt
@@ -652,30 +652,51 @@ sniffio==1.3.1 ; python_version >= "3.10" and python_version < "4.0" \
     --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \
     --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc
 sqlalchemy==1.4.51 ; python_version >= "3.10" and python_version < "4.0" \
+    --hash=sha256:0525c4905b4b52d8ccc3c203c9d7ab2a80329ffa077d4bacf31aefda7604dc65 \
     --hash=sha256:0535d5b57d014d06ceeaeffd816bb3a6e2dddeb670222570b8c4953e2d2ea678 \
     --hash=sha256:0892e7ac8bc76da499ad3ee8de8da4d7905a3110b952e2a35a940dab1ffa550e \
     --hash=sha256:0d661cff58c91726c601cc0ee626bf167b20cc4d7941c93c5f3ac28dc34ddbea \
+    --hash=sha256:1980e6eb6c9be49ea8f89889989127daafc43f0b1b6843d71efab1514973cca0 \
+    --hash=sha256:1a09d5bd1a40d76ad90e5570530e082ddc000e1d92de495746f6257dc08f166b \
     --hash=sha256:245c67c88e63f1523e9216cad6ba3107dea2d3ee19adc359597a628afcabfbcb \
     --hash=sha256:2ad16880ccd971ac8e570550fbdef1385e094b022d6fc85ef3ce7df400dddad3 \
     --hash=sha256:2be4e6294c53f2ec8ea36486b56390e3bcaa052bf3a9a47005687ccf376745d1 \
     --hash=sha256:2c55040d8ea65414de7c47f1a23823cd9f3fad0dc93e6b6b728fee81230f817b \
     --hash=sha256:352df882088a55293f621328ec33b6ffca936ad7f23013b22520542e1ab6ad1b \
     --hash=sha256:3823dda635988e6744d4417e13f2e2b5fe76c4bf29dd67e95f98717e1b094cad \
     --hash=sha256:38ef80328e3fee2be0a1abe3fe9445d3a2e52a1282ba342d0dab6edf1fef4707 \
+    --hash=sha256:39b02b645632c5fe46b8dd30755682f629ffbb62ff317ecc14c998c21b2896ff \
+    --hash=sha256:3b0cd89a7bd03f57ae58263d0f828a072d1b440c8c2949f38f3b446148321171 \
     --hash=sha256:3ec7a0ed9b32afdf337172678a4a0e6419775ba4e649b66f49415615fa47efbd \
+    --hash=sha256:3f0ef620ecbab46e81035cf3dedfb412a7da35340500ba470f9ce43a1e6c423b \
+    --hash=sha256:50e074aea505f4427151c286955ea025f51752fa42f9939749336672e0674c81 \
     --hash=sha256:55e699466106d09f028ab78d3c2e1f621b5ef2c8694598242259e4515715da7c \
+    --hash=sha256:5e180fff133d21a800c4f050733d59340f40d42364fcb9d14f6a67764bdc48d2 \
     --hash=sha256:6cacc0b2dd7d22a918a9642fc89840a5d3cee18a0e1fe41080b1141b23b10916 \
+    --hash=sha256:7af40425ac535cbda129d9915edcaa002afe35d84609fd3b9d6a8c46732e02ee \
+    --hash=sha256:7d8139ca0b9f93890ab899da678816518af74312bb8cd71fb721436a93a93298 \
     --hash=sha256:7deeae5071930abb3669b5185abb6c33ddfd2398f87660fafdb9e6a5fb0f3f2f \
     --hash=sha256:86a22143a4001f53bf58027b044da1fb10d67b62a785fc1390b5c7f089d9838c \
     --hash=sha256:8ca484ca11c65e05639ffe80f20d45e6be81fbec7683d6c9a15cd421e6e8b340 \
+    --hash=sha256:8d1d7d63e5d2f4e92a39ae1e897a5d551720179bb8d1254883e7113d3826d43c \
+    --hash=sha256:8e702e7489f39375601c7ea5a0bef207256828a2bc5986c65cb15cd0cf097a87 \
+    --hash=sha256:a055ba17f4675aadcda3005df2e28a86feb731fdcc865e1f6b4f209ed1225cba \
+    --hash=sha256:a33cb3f095e7d776ec76e79d92d83117438b6153510770fcd57b9c96f9ef623d \
+    --hash=sha256:a61184c7289146c8cff06b6b41807c6994c6d437278e72cf00ff7fe1c7a263d1 \
     --hash=sha256:af55cc207865d641a57f7044e98b08b09220da3d1b13a46f26487cc2f898a072 \
+    --hash=sha256:b00cf0471888823b7a9f722c6c41eb6985cf34f077edcf62695ac4bed6ec01ee \
+    --hash=sha256:b03850c290c765b87102959ea53299dc9addf76ca08a06ea98383348ae205c99 \
     --hash=sha256:b97fd5bb6b7c1a64b7ac0632f7ce389b8ab362e7bd5f60654c2a418496be5d7f \
     --hash=sha256:c37bc677690fd33932182b85d37433845de612962ed080c3e4d92f758d1bd894 \
+    --hash=sha256:cecb66492440ae8592797dd705a0cbaa6abe0555f4fa6c5f40b078bd2740fc6b \
+    --hash=sha256:d0a83afab5e062abffcdcbcc74f9d3ba37b2385294dd0927ad65fc6ebe04e054 \
     --hash=sha256:d3cf56cc36d42908495760b223ca9c2c0f9f0002b4eddc994b24db5fcb86a9e4 \
     --hash=sha256:e646b19f47d655261b22df9976e572f588185279970efba3d45c377127d35349 \
     --hash=sha256:e7908c2025eb18394e32d65dd02d2e37e17d733cdbe7d78231c2b6d7eb20cdb9 \
     --hash=sha256:e8f2df79a46e130235bc5e1bbef4de0583fb19d481eaa0bffa76e8347ea45ec6 \
+    --hash=sha256:eaeeb2464019765bc4340214fca1143081d49972864773f3f1e95dba5c7edc7d \
     --hash=sha256:eb18549b770351b54e1ab5da37d22bc530b8bfe2ee31e22b9ebe650640d2ef12 \
+    --hash=sha256:f2e5b6f5cf7c18df66d082604a1d9c7a2d18f7d1dbe9514a2afaccbb51cc4fc3 \
     --hash=sha256:f8cafa6f885a0ff5e39efa9325195217bb47d5929ab0051636610d24aef45ade
 starlette==0.37.2 ; python_version >= "3.10" and python_version < "4.0" \
     --hash=sha256:6fe59f29268538e5d0d182f2791a479a0c64638e6935d1c6989e63fb2699c6ee \

diff --git a/bytes/requirements.txt b/bytes/requirements.txt
@@ -637,30 +637,51 @@ sniffio==1.3.1 ; python_version >= "3.10" and python_version < "4.0" \
     --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \
     --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc
 sqlalchemy==1.4.51 ; python_version >= "3.10" and python_version < "4.0" \
+    --hash=sha256:0525c4905b4b52d8ccc3c203c9d7ab2a80329ffa077d4bacf31aefda7604dc65 \
     --hash=sha256:0535d5b57d014d06ceeaeffd816bb3a6e2dddeb670222570b8c4953e2d2ea678 \
     --hash=sha256:0892e7ac8bc76da499ad3ee8de8da4d7905a3110b952e2a35a940dab1ffa550e \
     --hash=sha256:0d661cff58c91726c601cc0ee626bf167b20cc4d7941c93c5f3ac28dc34ddbea \
+    --hash=sha256:1980e6eb6c9be49ea8f89889989127daafc43f0b1b6843d71efab1514973cca0 \
+    --hash=sha256:1a09d5bd1a40d76ad90e5570530e082ddc000e1d92de495746f6257dc08f166b \
     --hash=sha256:245c67c88e63f1523e9216cad6ba3107dea2d3ee19adc359597a628afcabfbcb \
     --hash=sha256:2ad16880ccd971ac8e570550fbdef1385e094b022d6fc85ef3ce7df400dddad3 \
     --hash=sha256:2be4e6294c53f2ec8ea36486b56390e3bcaa052bf3a9a47005687ccf376745d1 \
     --hash=sha256:2c55040d8ea65414de7c47f1a23823cd9f3fad0dc93e6b6b728fee81230f817b \
     --hash=sha256:352df882088a55293f621328ec33b6ffca936ad7f23013b22520542e1ab6ad1b \
     --hash=sha256:3823dda635988e6744d4417e13f2e2b5fe76c4bf29dd67e95f98717e1b094cad \
     --hash=sha256:38ef80328e3fee2be0a1abe3fe9445d3a2e52a1282ba342d0dab6edf1fef4707 \
+    --hash=sha256:39b02b645632c5fe46b8dd30755682f629ffbb62ff317ecc14c998c21b2896ff \
+    --hash=sha256:3b0cd89a7bd03f57ae58263d0f828a072d1b440c8c2949f38f3b446148321171 \
     --hash=sha256:3ec7a0ed9b32afdf337172678a4a0e6419775ba4e649b66f49415615fa47efbd \
+    --hash=sha256:3f0ef620ecbab46e81035cf3dedfb412a7da35340500ba470f9ce43a1e6c423b \
+    --hash=sha256:50e074aea505f4427151c286955ea025f51752fa42f9939749336672e0674c81 \
     --hash=sha256:55e699466106d09f028ab78d3c2e1f621b5ef2c8694598242259e4515715da7c \
+    --hash=sha256:5e180fff133d21a800c4f050733d59340f40d42364fcb9d14f6a67764bdc48d2 \
     --hash=sha256:6cacc0b2dd7d22a918a9642fc89840a5d3cee18a0e1fe41080b1141b23b10916 \
+    --hash=sha256:7af40425ac535cbda129d9915edcaa002afe35d84609fd3b9d6a8c46732e02ee \
+    --hash=sha256:7d8139ca0b9f93890ab899da678816518af74312bb8cd71fb721436a93a93298 \
     --hash=sha256:7deeae5071930abb3669b5185abb6c33ddfd2398f87660fafdb9e6a5fb0f3f2f \
     --hash=sha256:86a22143a4001f53bf58027b044da1fb10d67b62a785fc1390b5c7f089d9838c \
     --hash=sha256:8ca484ca11c65e05639ffe80f20d45e6be81fbec7683d6c9a15cd421e6e8b340 \
+    --hash=sha256:8d1d7d63e5d2f4e92a39ae1e897a5d551720179bb8d1254883e7113d3826d43c \
+    --hash=sha256:8e702e7489f39375601c7ea5a0bef207256828a2bc5986c65cb15cd0cf097a87 \
+    --hash=sha256:a055ba17f4675aadcda3005df2e28a86feb731fdcc865e1f6b4f209ed1225cba \
+    --hash=sha256:a33cb3f095e7d776ec76e79d92d83117438b6153510770fcd57b9c96f9ef623d \
+    --hash=sha256:a61184c7289146c8cff06b6b41807c6994c6d437278e72cf00ff7fe1c7a263d1 \
     --hash=sha256:af55cc207865d641a57f7044e98b08b09220da3d1b13a46f26487cc2f898a072 \
+    --hash=sha256:b00cf0471888823b7a9f722c6c41eb6985cf34f077edcf62695ac4bed6ec01ee \
+    --hash=sha256:b03850c290c765b87102959ea53299dc9addf76ca08a06ea98383348ae205c99 \
     --hash=sha256:b97fd5bb6b7c1a64b7ac0632f7ce389b8ab362e7bd5f60654c2a418496be5d7f \
     --hash=sha256:c37bc677690fd33932182b85d37433845de612962ed080c3e4d92f758d1bd894 \
+    --hash=sha256:cecb66492440ae8592797dd705a0cbaa6abe0555f4fa6c5f40b078bd2740fc6b \
+    --hash=sha256:d0a83afab5e062abffcdcbcc74f9d3ba37b2385294dd0927ad65fc6ebe04e054 \
     --hash=sha256:d3cf56cc36d42908495760b223ca9c2c0f9f0002b4eddc994b24db5fcb86a9e4 \
     --hash=sha256:e646b19f47d655261b22df9976e572f588185279970efba3d45c377127d35349 \
     --hash=sha256:e7908c2025eb18394e32d65dd02d2e37e17d733cdbe7d78231c2b6d7eb20cdb9 \
     --hash=sha256:e8f2df79a46e130235bc5e1bbef4de0583fb19d481eaa0bffa76e8347ea45ec6 \
+    --hash=sha256:eaeeb2464019765bc4340214fca1143081d49972864773f3f1e95dba5c7edc7d \
     --hash=sha256:eb18549b770351b54e1ab5da37d22bc530b8bfe2ee31e22b9ebe650640d2ef12 \
+    --hash=sha256:f2e5b6f5cf7c18df66d082604a1d9c7a2d18f7d1dbe9514a2afaccbb51cc4fc3 \
     --hash=sha256:f8cafa6f885a0ff5e39efa9325195217bb47d5929ab0051636610d24aef45ade
 starlette==0.37.2 ; python_version >= "3.10" and python_version < "4.0" \
     --hash=sha256:6fe59f29268538e5d0d182f2791a479a0c64638e6935d1c6989e63fb2699c6ee \

diff --git a/bytes/tests/client.py b/bytes/tests/client.py
@@ -6,7 +6,7 @@
 from uuid import UUID
 
 import httpx
-from httpx import HTTPError
+from httpx import HTTPError, HTTPStatusError
 
 from bytes.api.models import BoefjeOutput
 from bytes.models import BoefjeMeta, NormalizerMeta
@@ -23,7 +23,7 @@ def wrapper(self, *args, **kwargs):  # type: ignore
         try:
             return function(self, *args, **kwargs)
         except HTTPError as error:
-            if error.response.status_code != 401:
+            if not isinstance(error, HTTPStatusError) or error.response.status_code != 401:
                 raise
 
             self.login()

diff --git a/bytes/tests/integration/test_bytes_api.py b/bytes/tests/integration/test_bytes_api.py
@@ -236,6 +236,23 @@ def test_raw(bytes_api_client: BytesAPIClient, event_manager: RabbitMQEventManag
     assert str(boefje_meta.id) in body.decode()
 
 
+def test_raw_big(bytes_api_client: BytesAPIClient, event_manager: RabbitMQEventManager) -> None:
+    boefje_meta = get_boefje_meta()
+    bytes_api_client.save_boefje_meta(boefje_meta)
+
+    raw = b"test 123" * 100000
+    raw_id = bytes_api_client.save_raw(boefje_meta.id, raw)
+
+    retrieved_raw = bytes_api_client.get_raw(raw_id)
+
+    assert retrieved_raw == raw
+
+    method, properties, body = event_manager.connection.channel().basic_get("test__raw_file_received")
+    event_manager.connection.channel().basic_ack(method.delivery_tag)
+
+    assert str(boefje_meta.id) in body.decode()
+
+
 def test_save_raw_with_one_mime_type(bytes_api_client: BytesAPIClient) -> None:
     boefje_meta = get_boefje_meta(meta_id=uuid.uuid4())
     bytes_api_client.save_boefje_meta(boefje_meta)