Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add download_iau2000_file() #56

Merged
merged 10 commits into from
Oct 4, 2024
Merged

Add download_iau2000_file() #56

Changes from 3 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
5f39577
adjust variable names, logging levels, return fields
EugeneDu-GA Sep 11, 2024
5b087d1
Add and fix download_iau2000_file()
EugeneDu-GA Sep 11, 2024
d49ec26
Merge commit 'a5f90ff2a284bde3827b191363649071a83dc7df' into gn_downl…
EugeneDu-GA Sep 19, 2024
477158b
Add version option in download_product_from_cddis()
EugeneDu-GA Sep 25, 2024
683260c
Revert argunemt name changes to attempt_url_download()
EugeneDu-GA Sep 27, 2024
14dba0e
Fix empty return
EugeneDu-GA Sep 27, 2024
bf84cd8
Docstrings
EugeneDu-GA Sep 27, 2024
8eeaebe
Merge commit 'bd9c1e82d8486d2ef77abf905c01e07227e8d3fa' into gn_downl…
EugeneDu-GA Sep 27, 2024
c9c6203
,
EugeneDu-GA Sep 27, 2024
a20fdb5
Formatting
EugeneDu-GA Oct 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 78 additions & 23 deletions gnssanalysis/gn_download.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -503,27 +503,27 @@ def attempt_ftps_download(


def attempt_url_download(
download_dir: _Path, url: str, filename: str = None, type_of_file: str = None, if_file_present: str = "prompt_user"
download_dir: _Path, url: str, download_filename: str = None, type_of_file: str = None, if_file_present: str = "prompt_user"
EugeneDu-GA marked this conversation as resolved.
Show resolved Hide resolved
) -> _Path:
"""Attempt download of file given URL (url) to chosen location (download_dir)

:param _Path download_dir: Where to download files (local directory)
:param str url: URL to download
:param str filename: Filename to assign for the downloaded file, defaults to None
:param str download_filename: Filename to assign for the downloaded file, defaults to None
:param str type_of_file: How to label the file for STDOUT messages, defaults to None
:param str if_file_present: What to do if file already present: "replace", "dont_replace", defaults to "prompt_user"
:return _Path: The pathlib.Path of the downloaded file
"""
# If the filename is not provided, use the filename from the URL
if not filename:
filename = url[url.rfind("/") + 1 :]
logging.info(f"Attempting URL Download of {type_of_file} file - {filename} to {download_dir}")
# If the download_filename is not provided, use the filename from the URL
if not download_filename:
download_filename = url[url.rfind("/") + 1 :]
logging.info(f"Attempting URL Download of {type_of_file} file - {download_filename} to {download_dir}")
# Use the check_whether_to_download function to determine whether to download the file
download_filepath = check_whether_to_download(
filename=filename, download_dir=download_dir, if_file_present=if_file_present
filename=download_filename, download_dir=download_dir, if_file_present=if_file_present
)
if download_filepath:
download_url(url, download_filepath)
download_filepath = download_url(url, download_filepath)
return download_filepath


Expand Down Expand Up @@ -751,7 +751,7 @@ def download_file_from_cddis(
except _ftplib.all_errors as e:
retries += 1
if retries > max_retries:
logging.info(f"Failed to download {filename} and reached maximum retry count ({max_retries}).")
logging.error(f"Failed to download {filename} and reached maximum retry count ({max_retries}).")
if (output_folder / filename).is_file():
(output_folder / filename).unlink()
raise e
Expand Down Expand Up @@ -787,7 +787,7 @@ def download_product_from_cddis(
project_type: str = "OPS",
timespan: _datetime.timedelta = _datetime.timedelta(days=2),
if_file_present: str = "prompt_user",
) -> None:
) -> List[_Path]:
"""Download the file/s from CDDIS based on start and end epoch, to the download directory (download_dir)

:param _Path download_dir: Where to download files (local directory)
Expand All @@ -803,6 +803,7 @@ def download_product_from_cddis(
:param _datetime.timedelta timespan: Timespan of the file/s to download, defaults to _datetime.timedelta(days=2)
:param str if_file_present: What to do if file already present: "replace", "dont_replace", defaults to "prompt_user"
:raises FileNotFoundError: Raise error if the specified file cannot be found on CDDIS
:return List[_Path]: Return list of download files
"""
# Download the correct IGS FIN ERP files
if file_ext == "ERP" and analysis_center == "IGS" and solution_type == "FIN": # get the correct start_epoch
Expand Down Expand Up @@ -830,12 +831,15 @@ def download_product_from_cddis(
logging.debug(
f"Generated filename: {product_filename}, with GPS Date: {gps_date.gpswkD} and reference: {reference_start}"
)

ensure_folders([download_dir])
download_filepaths = []
EugeneDu-GA marked this conversation as resolved.
Show resolved Hide resolved
with ftp_tls(CDDIS_FTP) as ftps:
try:
ftps.cwd(f"gnss/products/{gps_date.gpswk}")
except _ftplib.all_errors as e:
logging.info(f"{reference_start} too recent")
logging.info(f"ftp_lib error: {e}")
logging.warning(f"{reference_start} too recent")
logging.warning(f"ftp_lib error: {e}")
product_filename, gps_date, reference_start = generate_product_filename(
reference_start,
file_ext,
Expand All @@ -851,7 +855,7 @@ def download_product_from_cddis(

all_files = ftps.nlst()
if not (product_filename in all_files):
logging.info(f"{product_filename} not in gnss/products/{gps_date.gpswk} - too recent")
logging.warning(f"{product_filename} not in gnss/products/{gps_date.gpswk} - too recent")
raise FileNotFoundError

# reference_start will be changed in the first run through while loop below
Expand All @@ -877,17 +881,68 @@ def download_product_from_cddis(
filename=product_filename, download_dir=download_dir, if_file_present=if_file_present
)
if download_filepath:
download_file_from_cddis(
filename=product_filename,
ftp_folder=f"gnss/products/{gps_date.gpswk}",
output_folder=download_dir,
if_file_present=if_file_present,
note_filetype=file_ext,
download_filepaths.append(
download_file_from_cddis(
filename=product_filename,
ftp_folder=f"gnss/products/{gps_date.gpswk}",
output_folder=download_dir,
if_file_present=if_file_present,
note_filetype=file_ext,
)
)
count += 1
remain = end_epoch - reference_start


def download_iau2000_file(download_dir: _Path, start_epoch: _datetime, if_file_present: str = "prompt_user"):
"""
Download relevant IAU2000 file from CDDIS or IERS based on start_epoch of data
EugeneDu-GA marked this conversation as resolved.
Show resolved Hide resolved
"""
ensure_folders([download_dir])
# Download most recent daily IAU2000 file if running for a session within the past week (data is within 3 months)
if _datetime.datetime.now() - start_epoch < _datetime.timedelta(weeks=1):
url_dir = "daily/"
iau2000_filename = "finals2000A.daily"
download_filename = "finals.daily.iau2000.txt"
logging.info("Attempting Download of finals2000A.daily file")
# Otherwise download the IAU2000 file dating back to 1992
else:
url_dir = "standard/"
iau2000_filename = "finals2000A.data"
download_filename = "finals.data.iau2000.txt"
logging.info("Attempting Download of finals2000A.data file")
filetype = "EOP IAU2000"

if not check_whether_to_download(
filename=download_filename, download_dir=download_dir, if_file_present=if_file_present
):
return None

# Attempt download from the CDDIS website first, if that fails try IERS
# Eugene: should try IERS first and then CDDIS?
try:
logging.info("Downloading IAU2000 file from CDDIS")
download_filepath = download_file_from_cddis(
filename=iau2000_filename,
ftp_folder="products/iers/",
output_folder=download_dir,
decompress=False,
if_file_present=if_file_present,
note_filetype=filetype
)
download_filepath = download_filepath.rename(download_dir / download_filename)
except:
logging.info("Failed CDDIS download - Downloading IAU2000 file from IERS")
download_filepath = attempt_url_download(
download_dir=download_dir,
url="https://datacenter.iers.org/products/eop/rapid/" + url_dir + iau2000_filename,
download_filename=download_filename,
type_of_file=filetype,
if_file_present=if_file_present,
)
return download_filepath


def download_atx(download_dir: _Path, reference_frame: str = "IGS20", if_file_present: str = "prompt_user") -> _Path:
"""Download the ATX file necessary for running the PEA provided the download directory (download_dir)

Expand All @@ -912,15 +967,15 @@ def download_atx(download_dir: _Path, reference_frame: str = "IGS20", if_file_pr
download_filepath = attempt_url_download(
download_dir=download_dir,
url=url_igs,
filename=atx_filename,
download_filename=atx_filename,
type_of_file="ATX",
if_file_present=if_file_present,
)
except:
download_filepath = attempt_url_download(
download_dir=download_dir,
url=url_bern,
filename=atx_filename,
download_filename=atx_filename,
type_of_file="ATX",
if_file_present=if_file_present,
)
Expand All @@ -938,7 +993,7 @@ def download_satellite_metadata_snx(download_dir: _Path, if_file_present: str =
download_filepath = attempt_url_download(
download_dir=download_dir,
url=IGS_FILES_URL + "station/general/igs_satellite_metadata.snx",
filename="igs_satellite_metadata.snx",
download_filename="igs_satellite_metadata.snx",
type_of_file="IGS satellite metadata",
if_file_present=if_file_present,
)
Expand All @@ -959,7 +1014,7 @@ def download_yaw_files(download_dir: _Path, if_file_present: str = "prompt_user"
download_filepath = attempt_url_download(
download_dir=download_dir,
url=PRODUCT_BASE_URL + "tables/" + filename,
filename=filename,
download_filename=filename,
type_of_file="Yaw Model SNX",
if_file_present=if_file_present,
)
Expand Down
Loading