From 95aee14989c1e0b10547bb354245170770dd8e45 Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Thu, 19 Oct 2023 18:19:34 +0100 Subject: [PATCH 1/9] nfc: I just re-ordered the imports --- tom_dataproducts/forced_photometry/atlas.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tom_dataproducts/forced_photometry/atlas.py b/tom_dataproducts/forced_photometry/atlas.py index 062fcdce4..5e9011217 100644 --- a/tom_dataproducts/forced_photometry/atlas.py +++ b/tom_dataproducts/forced_photometry/atlas.py @@ -1,7 +1,9 @@ +from datetime import timedelta, datetime +from astropy.time import Time +from crispy_forms.layout import Div, HTML from django import forms from django.conf import settings -from crispy_forms.layout import Div, HTML -from astropy.time import Time + import tom_dataproducts.forced_photometry.forced_photometry_service as fps from tom_dataproducts.tasks import atlas_query from tom_targets.models import Target From 1ac3a4df882a11dbde92a8721414c2e391aa8297 Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Thu, 19 Oct 2023 18:20:28 +0100 Subject: [PATCH 2/9] provides reasonable-ish initial values for the ATLAS query timebox --- tom_dataproducts/forced_photometry/atlas.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tom_dataproducts/forced_photometry/atlas.py b/tom_dataproducts/forced_photometry/atlas.py index 5e9011217..96b478d0a 100644 --- a/tom_dataproducts/forced_photometry/atlas.py +++ b/tom_dataproducts/forced_photometry/atlas.py @@ -29,6 +29,10 @@ class AtlasForcedPhotometryQueryForm(fps.BaseForcedPhotometryQueryForm): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + # initialize query time range to reasonable values + now = datetime.now() + self.fields['max_date'].initial = (now - timedelta(minutes=1)).strftime('%Y-%m-%dT%H:%M') + self.fields['min_date'].initial = (now - timedelta(days=20)).strftime('%Y-%m-%dT%H:%M') def layout(self): return Div( From 2f7851cfc282947a294be18c5d61af9cbb5d7426 Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Thu, 19 Oct 2023 18:21:31 +0100 Subject: [PATCH 3/9] I was getting a stacktrace b/c add_error wants two params The first, the field that the error pertains to, can be None. --- tom_dataproducts/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index b4d7f7776..3904f2bce 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -151,7 +151,7 @@ def post(self, request, *args, **kwargs): try: service.query_service(form.cleaned_data) except fps.ForcedPhotometryServiceException as e: - form.add_error(f"Problem querying forced photometry service: {repr(e)}") + form.add_error(None, f"Problem querying forced photometry service: {repr(e)}") return self.form_invalid(form) messages.info(self.request, service.get_success_message()) return redirect( From f188e9be9a36632a5759195d69de8669c56893d7 Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Thu, 19 Oct 2023 22:06:26 +0100 Subject: [PATCH 4/9] reminder that dramatiq workers must be running for asynch operation --- docs/managing_data/forced_photometry.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/managing_data/forced_photometry.rst b/docs/managing_data/forced_photometry.rst index a9e9f7665..884bdd138 100644 --- a/docs/managing_data/forced_photometry.rst +++ b/docs/managing_data/forced_photometry.rst @@ -72,8 +72,9 @@ a redis server, you would add the following to your ``settings.py``: After adding the ``django_dramatiq`` installed app, you will need to run ``./manage.py migrate`` once to setup its DB tables. If this configuration is set in your TOM, the existing services which support asynchronous queries, -Atlas and ZTF, should start querying asynchronously. If you do not add these settings, those services will still -function but will fall back to synchronous queries. +Atlas and ZTF, should start querying asynchronously. (Note: You must also start the dramatiq workers: +``./manage.py rundramatic``. If you do not add these settings, those services will still function but will fall +back to synchronous queries. Adding a new Forced Photometry Service From de641a4222b8e0177b4cf23abab8e20099c2378f Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Fri, 20 Oct 2023 01:29:54 +0100 Subject: [PATCH 5/9] this is more consistent with TOM Toolkit naming conventions --- docs/managing_data/forced_photometry.rst | 2 +- tom_dataproducts/tasks.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/managing_data/forced_photometry.rst b/docs/managing_data/forced_photometry.rst index 884bdd138..8453236d4 100644 --- a/docs/managing_data/forced_photometry.rst +++ b/docs/managing_data/forced_photometry.rst @@ -14,7 +14,7 @@ shown below: .. code:: python FORCED_PHOTOMETRY_SERVICES = { - 'atlas': { + 'ATLAS': { 'class': 'tom_dataproducts.forced_photometry.atlas.AtlasForcedPhotometryService', 'url': "https://fallingstar-data.com/forcedphot", 'api_key': os.getenv('ATLAS_FORCED_PHOTOMETRY_API_KEY', 'your atlas account api token') diff --git a/tom_dataproducts/tasks.py b/tom_dataproducts/tasks.py index f6ce1b2d9..d294be029 100644 --- a/tom_dataproducts/tasks.py +++ b/tom_dataproducts/tasks.py @@ -23,9 +23,9 @@ def atlas_query(min_date_mjd, max_date_mjd, target_id, data_product_type): print("Calling atlas query!") target = Target.objects.get(pk=target_id) - headers = {"Authorization": f"Token {settings.FORCED_PHOTOMETRY_SERVICES.get('atlas', {}).get('api_key')}", + headers = {"Authorization": f"Token {settings.FORCED_PHOTOMETRY_SERVICES.get('ATLAS', {}).get('api_key')}", "Accept": "application/json"} - base_url = settings.FORCED_PHOTOMETRY_SERVICES.get('atlas', {}).get('url') + base_url = settings.FORCED_PHOTOMETRY_SERVICES.get('ATLAS', {}).get('url') task_url = None while not task_url: with requests.Session() as s: From 29d05a6821e0606decb86c7ad21b3df8929b3c2f Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Fri, 20 Oct 2023 01:30:55 +0100 Subject: [PATCH 6/9] make all those print statements logger.debug statements --- tom_dataproducts/tasks.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tom_dataproducts/tasks.py b/tom_dataproducts/tasks.py index d294be029..9a2473c6b 100644 --- a/tom_dataproducts/tasks.py +++ b/tom_dataproducts/tasks.py @@ -17,11 +17,11 @@ from tom_dataproducts.data_processor import run_data_processor logger = logging.getLogger(__name__) - +logger.setLevel(logging.DEBUG) @dramatiq.actor(max_retries=0) def atlas_query(min_date_mjd, max_date_mjd, target_id, data_product_type): - print("Calling atlas query!") + logger.debug('Calling atlas query!') target = Target.objects.get(pk=target_id) headers = {"Authorization": f"Token {settings.FORCED_PHOTOMETRY_SERVICES.get('ATLAS', {}).get('api_key')}", "Accept": "application/json"} @@ -38,10 +38,10 @@ def atlas_query(min_date_mjd, max_date_mjd, target_id, data_product_type): if resp.status_code == 201: task_url = resp.json()["url"] - print(f"The task url is {task_url}") + logger.debug(f"The task url is {task_url}") elif resp.status_code == 429: message = resp.json()["detail"] - print(f"{resp.status_code} {message}") + logger.debug(f"{resp.status_code} {message}") t_sec = re.findall(r"available in (\d+) seconds", message) t_min = re.findall(r"available in (\d+) minutes", message) if t_sec: @@ -50,7 +50,7 @@ def atlas_query(min_date_mjd, max_date_mjd, target_id, data_product_type): waittime = int(t_min[0]) * 60 else: waittime = 10 - print(f"Waiting {waittime} seconds") + logger.debug(f"Waiting {waittime} seconds") time.sleep(waittime) else: logger.error(f"Failed to queue Atlas task: HTTP Error {resp.status_code} - {resp.text}") @@ -65,14 +65,14 @@ def atlas_query(min_date_mjd, max_date_mjd, target_id, data_product_type): if resp.status_code == 200: if resp.json()["finishtimestamp"]: result_url = resp.json()["result_url"] # PART WHEN QUERY IS COMPLETE - print(f"Task is complete with results available at {result_url}") + logger.debug(f"Task is complete with results available at {result_url}") elif resp.json()["starttimestamp"]: if not taskstarted_printed: - print(f"Task is running (started at {resp.json()['starttimestamp']})") + logger.debug(f"Task is running (started at {resp.json()['starttimestamp']})") taskstarted_printed = True time.sleep(2) else: - print(f"Waiting for job to start (queued at {resp.json()['timestamp']})") + logger.debug(f"Waiting for job to start (queued at {resp.json()['timestamp']})") time.sleep(4) else: logger.error(f"Failed to retrieve Atlas task status: HTTP Error {resp.status_code} - {resp.text}") From a11d146c4325eaf4734975b7d41581b33682c4ce Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Fri, 20 Oct 2023 01:31:30 +0100 Subject: [PATCH 7/9] we crash with empyt config dictionaries, so leave them out --- docs/managing_data/forced_photometry.rst | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/managing_data/forced_photometry.rst b/docs/managing_data/forced_photometry.rst index 8453236d4..1c3def422 100644 --- a/docs/managing_data/forced_photometry.rst +++ b/docs/managing_data/forced_photometry.rst @@ -19,12 +19,11 @@ shown below: 'url': "https://fallingstar-data.com/forcedphot", 'api_key': os.getenv('ATLAS_FORCED_PHOTOMETRY_API_KEY', 'your atlas account api token') }, - 'panstarrs': { - #TODO - }, - 'ztf': { - #TODO - } + # TODO: these services are coming soon... + # 'PANSTARSS': { + # }, + # 'ZTF': { + # } } DATA_PRODUCT_TYPES = { From a85d459c375a0ab0c84fedd1214642fd423bc3c7 Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Fri, 20 Oct 2023 01:40:19 +0100 Subject: [PATCH 8/9] missed a few 'atlas' to 'ATLAS' changes --- tom_dataproducts/forced_photometry/atlas.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tom_dataproducts/forced_photometry/atlas.py b/tom_dataproducts/forced_photometry/atlas.py index 96b478d0a..5447e7807 100644 --- a/tom_dataproducts/forced_photometry/atlas.py +++ b/tom_dataproducts/forced_photometry/atlas.py @@ -109,15 +109,15 @@ def query_service(self, query_parameters): if not Target.objects.filter(pk=query_parameters.get('target_id')).exists(): raise fps.ForcedPhotometryServiceException(f"Target {query_parameters.get('target_id')} does not exist") - if 'atlas' not in settings.FORCED_PHOTOMETRY_SERVICES: - raise fps.ForcedPhotometryServiceException("Must specify 'atlas' settings in FORCED_PHOTOMETRY_SERVICES") - if not settings.FORCED_PHOTOMETRY_SERVICES.get('atlas', {}).get('url'): + if 'ATLAS' not in settings.FORCED_PHOTOMETRY_SERVICES: + raise fps.ForcedPhotometryServiceException("Must specify 'ATLAS' settings in FORCED_PHOTOMETRY_SERVICES") + if not settings.FORCED_PHOTOMETRY_SERVICES.get('ATLAS', {}).get('url'): raise fps.ForcedPhotometryServiceException( - "Must specify a 'url' under atlas settings in FORCED_PHOTOMETRY_SERVICES" + "Must specify a 'url' under ATLAS settings in FORCED_PHOTOMETRY_SERVICES" ) - if not settings.FORCED_PHOTOMETRY_SERVICES.get('atlas', {}).get('api_key'): + if not settings.FORCED_PHOTOMETRY_SERVICES.get('ATLAS', {}).get('api_key'): raise fps.ForcedPhotometryServiceException( - "Must specify an 'api_key' under atlas settings in FORCED_PHOTOMETRY_SERVICES" + "Must specify an 'api_key' under ATLAS settings in FORCED_PHOTOMETRY_SERVICES" ) if 'django_dramatiq' in settings.INSTALLED_APPS: From b45de0473682e9917fa1753ae7a110873da1030f Mon Sep 17 00:00:00 2001 From: "William (Lindy) Lindstrom" Date: Fri, 20 Oct 2023 01:45:55 +0100 Subject: [PATCH 9/9] flake8 compliance --- tom_dataproducts/tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tom_dataproducts/tasks.py b/tom_dataproducts/tasks.py index 9a2473c6b..d93783fd3 100644 --- a/tom_dataproducts/tasks.py +++ b/tom_dataproducts/tasks.py @@ -19,6 +19,7 @@ logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) + @dramatiq.actor(max_retries=0) def atlas_query(min_date_mjd, max_date_mjd, target_id, data_product_type): logger.debug('Calling atlas query!')