Skip to content

Commit

Permalink
add timeout to download function
Browse files Browse the repository at this point in the history
  • Loading branch information
semio committed Apr 2, 2020
1 parent dff8632 commit f33916a
Showing 1 changed file with 7 additions and 4 deletions.
11 changes: 7 additions & 4 deletions ddf_utils/factory/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,8 @@ def newfunc(*args, **kwargs):
return wrapper


def download(url, out_file, session=None, resume=True, method="GET", post_data=None, retry_times=5, backoff=0.5, progress_bar=True):
def download(url, out_file, session=None, resume=True, method="GET", post_data=None,
retry_times=5, backoff=0.5, progress_bar=True, timeout=10):
"""Download a url, and optionally try to resume it.
Parameters
Expand All @@ -81,11 +82,13 @@ def download(url, out_file, session=None, resume=True, method="GET", post_data=N
backoff : float
progress_bar : bool
whether to display a progress bar
timeout : int
maximum time to wait for connect/read server responses. (Note: not the time limit for total response)
"""

@retry(times=retry_times, backoff=backoff)
def get_file_size(request, session_):
response = session_.send(request, stream=True)
response = session_.send(request, stream=True, timeout=timeout)
response.raise_for_status()
return int(response.headers['Content-length'])

Expand All @@ -103,11 +106,11 @@ def run(request, session_, out_file_, resume_, file_size):
print(f'resumming {out_file_}...')
header = {"Range": f'bytes={first_byte}-{file_size}'}
request.headers = header
response = session_.send(request, stream=True)
response = session_.send(request, stream=True, timeout=timeout)
response.raise_for_status()
else:
print(f'begin downloading {out_file_}...')
response = session_.send(request, stream=True)
response = session_.send(request, stream=True, timeout=timeout)
response.raise_for_status()

if progress_bar:
Expand Down

0 comments on commit f33916a

Please sign in to comment.