Skip to content

Commit

Permalink
fix masking of ClientResponseError where some tool raises a generic E…
Browse files Browse the repository at this point in the history
…xception instead of the original ClientResponseError
  • Loading branch information
ungarj committed Dec 3, 2024
1 parent df95f33 commit 054ca7e
Showing 1 changed file with 22 additions and 11 deletions.
33 changes: 22 additions & 11 deletions mapchete/io/_misc.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import logging
from enum import Enum

from aiohttp import ClientResponseError
import rasterio
from rasterio.warp import calculate_default_transform
from shapely.errors import TopologicalError
Expand Down Expand Up @@ -196,14 +197,24 @@ def copy(src_path, dst_path, src_fs=None, dst_fs=None, overwrite=False):
# create parent directories on local filesystems
dst_path.parent.makedirs()

# copy either within a filesystem or between filesystems
if src_path.fs == dst_path.fs:
src_path.fs.copy(str(src_path), str(dst_path))
else:
# read source data first
with src_path.open("rb") as src:
content = src.read()
# only write to destination if reading source data didn't raise errors,
# otherwise we can end up with empty objects on an object store
with dst_path.open("wb") as dst:
dst.write(content)
try:
# copy either within a filesystem or between filesystems
if src_path.fs == dst_path.fs:
src_path.fs.copy(str(src_path), str(dst_path))
else:
# read source data first
with src_path.open("rb") as src:
content = src.read()
# only write to destination if reading source data didn't raise errors,
# otherwise we can end up with empty objects on an object store
with dst_path.open("wb") as dst:
dst.write(content)
except Exception as exception:
# This is a hack because some tool using aiohttp does not raise a
# ClientResponseError directly but masks it as a generic Exception and thus
# preventing our retry mechanism to kick in.
if repr(exception).startswith(
'Exception("ClientResponseError'
): # pragma: no cover
raise ClientResponseError(exception) from exception
raise

0 comments on commit 054ca7e

Please sign in to comment.