Skip to content

Commit

Permalink
improvements to SIA Download code
Browse files Browse the repository at this point in the history
  • Loading branch information
fccoelho committed Feb 3, 2022
1 parent 525621e commit 26133ea
Showing 1 changed file with 18 additions and 13 deletions.
31 changes: 18 additions & 13 deletions pysus/online_data/SIA.py
Original file line number Diff line number Diff line change
Expand Up @@ -138,20 +138,12 @@ def _fetch_file(fname, ftp, ftype):
"""

multiples = False
try:
fnames = check_file_split(fname, ftp)
if len(fnames) > 1:
multiples = True
for fn in fnames:
fnfull = os.path.join(CACHEPATH, fn)
print(f"Downloading {fn}...")
fobj = open(fnfull, "wb")
ftp.retrbinary(f"RETR {fn}", fobj.write)
dbc2dbf(fnfull, fnfull.replace('.dbc', '.dbf'))
os.unlink(fnfull)
except Exception as exc:
raise Exception(f"Retrieval of file {fn} failed with the following error:\n {exc}")
fnames = check_file_split(fname, ftp)

multiples = len(fnames) > 1

if multiples:
download_multiples(fnames, ftp)
print(f"This download is split into the following files: {fnames}\n"
f"They have been downloaded in {CACHEPATH}.\n"
f"To load them, use the pysus.utilities.read_dbc_dbf function.")
Expand All @@ -162,6 +154,19 @@ def _fetch_file(fname, ftp, ftype):
return df


def download_multiples(fnames, ftp):
for fn in fnames:
fnfull = os.path.join(CACHEPATH, fn)
print(f"Downloading {fn}...")
fobj = open(fnfull, "wb")
try:
ftp.retrbinary(f"RETR {fn}", fobj.write)
dbc2dbf(fnfull, fnfull.replace('.dbc', '.dbf'))
os.unlink(fnfull)
except Exception as exc:
raise Exception(f"Retrieval of file {fn} failed with the following error:\n {exc}")


def check_file_split(fname: str, ftp: FTP) -> list:
"""
Check for split filenames. Sometimes when files are too large, they are split into multiple files ending in a, b, c, ...
Expand Down

0 comments on commit 26133ea

Please sign in to comment.