Skip to content

Commit

Permalink
Merge branch 'dev' into fix/tom_sharing_tweaks
Browse files Browse the repository at this point in the history
  • Loading branch information
@jchate6
jchate6 authored Oct 23, 2023
2 parents 8f7221e + 4cb3e50 commit f068566
Show file tree
Hide file tree
Showing 4 changed files with 34 additions and 27 deletions.
18 changes: 9 additions & 9 deletions docs/managing_data/forced_photometry.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,17 +14,16 @@ shown below:

.. code:: python
FORCED_PHOTOMETRY_SERVICES = {
'atlas': {
'ATLAS': {
'class': 'tom_dataproducts.forced_photometry.atlas.AtlasForcedPhotometryService',
'url': "https://fallingstar-data.com/forcedphot",
'api_key': os.getenv('ATLAS_FORCED_PHOTOMETRY_API_KEY', 'your atlas account api token')
},
'panstarrs': {
#TODO
},
'ztf': {
#TODO
}
# TODO: these services are coming soon...
# 'PANSTARSS': {
# },
# 'ZTF': {
# }
}
DATA_PRODUCT_TYPES = {
Expand Down Expand Up @@ -72,8 +71,9 @@ a redis server, you would add the following to your ``settings.py``:
After adding the ``django_dramatiq`` installed app, you will need to run ``./manage.py migrate`` once to setup
its DB tables. If this configuration is set in your TOM, the existing services which support asynchronous queries,
Atlas and ZTF, should start querying asynchronously. If you do not add these settings, those services will still
function but will fall back to synchronous queries.
Atlas and ZTF, should start querying asynchronously. (Note: You must also start the dramatiq workers:
``./manage.py rundramatic``. If you do not add these settings, those services will still function but will fall
back to synchronous queries.


Adding a new Forced Photometry Service
Expand Down
22 changes: 14 additions & 8 deletions tom_dataproducts/forced_photometry/atlas.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
from datetime import timedelta, datetime
from astropy.time import Time
from crispy_forms.layout import Div, HTML
from django import forms
from django.conf import settings
from crispy_forms.layout import Div, HTML
from astropy.time import Time

import tom_dataproducts.forced_photometry.forced_photometry_service as fps
from tom_dataproducts.tasks import atlas_query
from tom_targets.models import Target
Expand All @@ -27,6 +29,10 @@ class AtlasForcedPhotometryQueryForm(fps.BaseForcedPhotometryQueryForm):

def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# initialize query time range to reasonable values
now = datetime.now()
self.fields['max_date'].initial = (now - timedelta(minutes=1)).strftime('%Y-%m-%dT%H:%M')
self.fields['min_date'].initial = (now - timedelta(days=20)).strftime('%Y-%m-%dT%H:%M')

def layout(self):
return Div(
Expand Down Expand Up @@ -103,15 +109,15 @@ def query_service(self, query_parameters):
if not Target.objects.filter(pk=query_parameters.get('target_id')).exists():
raise fps.ForcedPhotometryServiceException(f"Target {query_parameters.get('target_id')} does not exist")

if 'atlas' not in settings.FORCED_PHOTOMETRY_SERVICES:
raise fps.ForcedPhotometryServiceException("Must specify 'atlas' settings in FORCED_PHOTOMETRY_SERVICES")
if not settings.FORCED_PHOTOMETRY_SERVICES.get('atlas', {}).get('url'):
if 'ATLAS' not in settings.FORCED_PHOTOMETRY_SERVICES:
raise fps.ForcedPhotometryServiceException("Must specify 'ATLAS' settings in FORCED_PHOTOMETRY_SERVICES")
if not settings.FORCED_PHOTOMETRY_SERVICES.get('ATLAS', {}).get('url'):
raise fps.ForcedPhotometryServiceException(
"Must specify a 'url' under atlas settings in FORCED_PHOTOMETRY_SERVICES"
"Must specify a 'url' under ATLAS settings in FORCED_PHOTOMETRY_SERVICES"
)
if not settings.FORCED_PHOTOMETRY_SERVICES.get('atlas', {}).get('api_key'):
if not settings.FORCED_PHOTOMETRY_SERVICES.get('ATLAS', {}).get('api_key'):
raise fps.ForcedPhotometryServiceException(
"Must specify an 'api_key' under atlas settings in FORCED_PHOTOMETRY_SERVICES"
"Must specify an 'api_key' under ATLAS settings in FORCED_PHOTOMETRY_SERVICES"
)

if 'django_dramatiq' in settings.INSTALLED_APPS:
Expand Down
19 changes: 10 additions & 9 deletions tom_dataproducts/tasks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,16 @@
from tom_dataproducts.data_processor import run_data_processor

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)


@dramatiq.actor(max_retries=0)
def atlas_query(min_date_mjd, max_date_mjd, target_id, data_product_type):
print("Calling atlas query!")
logger.debug('Calling atlas query!')
target = Target.objects.get(pk=target_id)
headers = {"Authorization": f"Token {settings.FORCED_PHOTOMETRY_SERVICES.get('atlas', {}).get('api_key')}",
headers = {"Authorization": f"Token {settings.FORCED_PHOTOMETRY_SERVICES.get('ATLAS', {}).get('api_key')}",
"Accept": "application/json"}
base_url = settings.FORCED_PHOTOMETRY_SERVICES.get('atlas', {}).get('url')
base_url = settings.FORCED_PHOTOMETRY_SERVICES.get('ATLAS', {}).get('url')
task_url = None
while not task_url:
with requests.Session() as s:
Expand All @@ -38,10 +39,10 @@ def atlas_query(min_date_mjd, max_date_mjd, target_id, data_product_type):

if resp.status_code == 201:
task_url = resp.json()["url"]
print(f"The task url is {task_url}")
logger.debug(f"The task url is {task_url}")
elif resp.status_code == 429:
message = resp.json()["detail"]
print(f"{resp.status_code} {message}")
logger.debug(f"{resp.status_code} {message}")
t_sec = re.findall(r"available in (\d+) seconds", message)
t_min = re.findall(r"available in (\d+) minutes", message)
if t_sec:
Expand All @@ -50,7 +51,7 @@ def atlas_query(min_date_mjd, max_date_mjd, target_id, data_product_type):
waittime = int(t_min[0]) * 60
else:
waittime = 10
print(f"Waiting {waittime} seconds")
logger.debug(f"Waiting {waittime} seconds")
time.sleep(waittime)
else:
logger.error(f"Failed to queue Atlas task: HTTP Error {resp.status_code} - {resp.text}")
Expand All @@ -65,14 +66,14 @@ def atlas_query(min_date_mjd, max_date_mjd, target_id, data_product_type):
if resp.status_code == 200:
if resp.json()["finishtimestamp"]:
result_url = resp.json()["result_url"] # PART WHEN QUERY IS COMPLETE
print(f"Task is complete with results available at {result_url}")
logger.debug(f"Task is complete with results available at {result_url}")
elif resp.json()["starttimestamp"]:
if not taskstarted_printed:
print(f"Task is running (started at {resp.json()['starttimestamp']})")
logger.debug(f"Task is running (started at {resp.json()['starttimestamp']})")
taskstarted_printed = True
time.sleep(2)
else:
print(f"Waiting for job to start (queued at {resp.json()['timestamp']})")
logger.debug(f"Waiting for job to start (queued at {resp.json()['timestamp']})")
time.sleep(4)
else:
logger.error(f"Failed to retrieve Atlas task status: HTTP Error {resp.status_code} - {resp.text}")
Expand Down
2 changes: 1 addition & 1 deletion tom_dataproducts/views.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ def post(self, request, *args, **kwargs):
try:
service.query_service(form.cleaned_data)
except fps.ForcedPhotometryServiceException as e:
form.add_error(f"Problem querying forced photometry service: {repr(e)}")
form.add_error(None, f"Problem querying forced photometry service: {repr(e)}")
return self.form_invalid(form)
messages.info(self.request, service.get_success_message())
return redirect(
Expand Down

0 comments on commit f068566

Please sign in to comment.