From c3e9baf7e537a2b2294fbb9763017438e86ee8ef Mon Sep 17 00:00:00 2001 From: "T. Andrew Manning" Date: Sun, 5 May 2024 14:19:05 -0500 Subject: [PATCH 1/3] Use temp file to track last NED request time and implement sempahore --- astro_ghost/NEDQueryFunctions.py | 42 ++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/astro_ghost/NEDQueryFunctions.py b/astro_ghost/NEDQueryFunctions.py index c452c39..52e81e4 100644 --- a/astro_ghost/NEDQueryFunctions.py +++ b/astro_ghost/NEDQueryFunctions.py @@ -1,13 +1,37 @@ from astropy import units as u from astropy.coordinates import SkyCoord from astroquery.ipac.ned import Ned -import re +from time import sleep +from datetime import datetime, timezone, timedelta +import re, os import numpy as np from astro_ghost.PS1QueryFunctions import find_all import warnings warnings.filterwarnings("ignore", category=UserWarning) +NED_TIME_SLEEP = 2 + + +def ned_rate_limited(): + def ned_update_request_time(ned_time_file): + with open(ned_time_file, 'w') as fp: + fp.write(datetime.now(timezone.utc).strftime()) + ned_time_file = '/tmp/ned_last_api_call' + delay = False + if os.path.exists(ned_time_file): + with open(ned_time_file) as fp: + ned_last_api_call = fp.read() + last_query = datetime.strptime(ned_last_api_call) + current_time = datetime.now(timezone.utc) + delay = current_time - last_query < timedelta(seconds=NED_TIME_SLEEP) + if not delay: + ned_update_request_time(ned_time_file) + else: + ned_update_request_time(ned_time_file) + return delay + + def getNEDSpectra(df, path, verbose=False): """Downloads NED spectra for the host galaxy, if it exists. @@ -23,7 +47,11 @@ def getNEDSpectra(df, path, verbose=False): transientNames = np.array(df.dropna(subset=['NED_name'])['TransientName']) for j in np.arange(len(hostNames)): try: - spectra = Ned.get_spectra(hostNames[j]) + while ned_rate_limited(): + print(f'Avoiding NED rate limit. Sleeping for {NED_TIME_SLEEP} seconds...') + sleep(NED_TIME_SLEEP) + else: + spectra = Ned.get_spectra(hostNames[j]) except: continue if spectra: @@ -82,9 +110,13 @@ def getNEDInfo(df): try: #query NED with a 2'' radius. - result_table = Ned.query_region(c, radius=(2/3600)*u.deg, equinox='J2000.0') - if len(result_table) > 0: - missingCounter = 0 + while ned_rate_limited(): + print(f'Avoiding NED rate limit. Sleeping for {NED_TIME_SLEEP} seconds...') + sleep(NED_TIME_SLEEP) + else: + result_table = Ned.query_region(c, radius=(2/3600)*u.deg, equinox='J2000.0') + if len(result_table) > 0: + missingCounter = 0 except: missingCounter += 1 if len(result_table) > 0: From bca5d35c2252195a2cf07fa4b06e0c0c83111d65 Mon Sep 17 00:00:00 2001 From: David Jones Date: Thu, 25 Jul 2024 16:56:02 -1000 Subject: [PATCH 2/3] re-trigger workflows --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index 6d7d4d9..a5caf4a 100644 --- a/README.rst +++ b/README.rst @@ -90,3 +90,4 @@ Note: This disclaimer was originally written by `PyCon talk `_, and was adapted by astro_ghost based on its use in the README file for the `MetPy project `_. + From 15590cd4dab39fc3463aaa77655af433d78c5e34 Mon Sep 17 00:00:00 2001 From: David Jones Date: Fri, 26 Jul 2024 08:54:55 -1000 Subject: [PATCH 3/3] datetime version issue fixed, using isoformat for all dates --- astro_ghost/NEDQueryFunctions.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/astro_ghost/NEDQueryFunctions.py b/astro_ghost/NEDQueryFunctions.py index 52e81e4..81d1a61 100644 --- a/astro_ghost/NEDQueryFunctions.py +++ b/astro_ghost/NEDQueryFunctions.py @@ -16,16 +16,17 @@ def ned_rate_limited(): def ned_update_request_time(ned_time_file): with open(ned_time_file, 'w') as fp: - fp.write(datetime.now(timezone.utc).strftime()) + fp.write(datetime.now(timezone.utc).isoformat()) ned_time_file = '/tmp/ned_last_api_call' delay = False if os.path.exists(ned_time_file): with open(ned_time_file) as fp: ned_last_api_call = fp.read() - last_query = datetime.strptime(ned_last_api_call) - current_time = datetime.now(timezone.utc) - delay = current_time - last_query < timedelta(seconds=NED_TIME_SLEEP) - if not delay: + if ned_last_api_call: + last_query = datetime.fromisoformat(ned_last_api_call) + current_time = datetime.now(timezone.utc) + delay = current_time - last_query < timedelta(seconds=NED_TIME_SLEEP) + if not delay or not ned_last_api_call: ned_update_request_time(ned_time_file) else: ned_update_request_time(ned_time_file) @@ -119,6 +120,7 @@ def getNEDInfo(df): missingCounter = 0 except: missingCounter += 1 + if len(result_table) > 0: #if Messier or NGC object, take that, otherwise take the closest object result_df = result_table.to_pandas()