From ddc1e39126bb9fc548a25b71c685909fa455d70c Mon Sep 17 00:00:00 2001 From: Lloyd Dakin Date: Mon, 2 Dec 2024 11:31:52 -0800 Subject: [PATCH 1/4] source catalog and line profile use source for images, handle missing ra, dec, add clientAlertExceptions to analysis endpoint --- .../datalab_session/analysis/line_profile.py | 12 ++--- .../analysis/source_catalog.py | 38 ++++++++++++--- datalab/datalab_session/views.py | 47 +++++++++++++------ 3 files changed, 69 insertions(+), 28 deletions(-) diff --git a/datalab/datalab_session/analysis/line_profile.py b/datalab/datalab_session/analysis/line_profile.py index 47b2729..02c10cb 100644 --- a/datalab/datalab_session/analysis/line_profile.py +++ b/datalab/datalab_session/analysis/line_profile.py @@ -20,7 +20,7 @@ def line_profile(input: dict): y2 (int): The y coordinate of the ending point } """ - fits_path = get_fits(input['basename']) + fits_path = get_fits(input['basename'], input['source']) sci_hdu = get_hdu(fits_path, 'SCI') @@ -51,16 +51,16 @@ def line_profile(input: dict): position_angle = coordinates.position_angle(start_sky_coord.ra, start_sky_coord.dec, end_sky_coord.ra, end_sky_coord.dec).deg except WcsError: - # no valid WCS solution + # Can't calculate these values without WCS start_coords = None end_coords = None position_angle = None + # fallback attempt at using pixscale to calculate the arcsec distance try: - # attempt using pixscale to calculate the angle arcsec_angle = len(line_profile) * sci_hdu.header["PIXSCALE"] - except KeyError as e: - # no valid WCS solution, and no pixscale + except KeyError: arcsec_angle = None - return {"line_profile": line_profile, "arcsec": arcsec_angle, "start_coords": start_coords, "end_coords": end_coords, "position_angle": position_angle} + line_profile_output = {"line_profile": line_profile, "arcsec": arcsec_angle, "start_coords": start_coords, "end_coords": end_coords, "position_angle": position_angle} + return line_profile_output diff --git a/datalab/datalab_session/analysis/source_catalog.py b/datalab/datalab_session/analysis/source_catalog.py index cb331bf..e856512 100644 --- a/datalab/datalab_session/analysis/source_catalog.py +++ b/datalab/datalab_session/analysis/source_catalog.py @@ -3,11 +3,28 @@ from datalab.datalab_session.file_utils import get_hdu, scale_points from datalab.datalab_session.s3_utils import get_fits +# Source catalog Function Definition +# ARGS: input (dict) +# input = { +# basename (str): The name of the file to analyze +# height (int): The height of the image +# width (int): The width of the image +# source (str): The source of the file +# } +# RETURNS: output (dict) +# output = [{ +# x (int): The x coordinate of the source +# y (int): The y coordinate of the source +# flux (int): The flux value of the source +# ra (float): The right ascension of the source +# dec (float): The declination of the source +# }] +# def source_catalog(input: dict): """ Returns a dict representing the source catalog data with x,y coordinates and flux values """ - fits_path = get_fits(input['basename']) + fits_path = get_fits(input['basename'], input['source']) cat_hdu = get_hdu(fits_path, 'CAT') sci_hdu = get_hdu(fits_path, 'SCI') @@ -19,23 +36,30 @@ def source_catalog(input: dict): x_points = cat_hdu.data["x"][:SOURCE_CATALOG_COUNT] y_points = cat_hdu.data["y"][:SOURCE_CATALOG_COUNT] flux = cat_hdu.data["flux"][:SOURCE_CATALOG_COUNT] - ra = cat_hdu.data["ra"][:SOURCE_CATALOG_COUNT] - dec = cat_hdu.data["dec"][:SOURCE_CATALOG_COUNT] + + # ra, dec values may or may not be present in the CAT hdu + if "ra" in cat_hdu.data.names and "dec" in cat_hdu.data.names: + ra = cat_hdu.data["ra"][:SOURCE_CATALOG_COUNT] + dec = cat_hdu.data["dec"][:SOURCE_CATALOG_COUNT] + else: + # TODO: implement a fallback way to calculate ra, dec from x, y and WCS + ra = np.zeros(SOURCE_CATALOG_COUNT) + dec = np.zeros(SOURCE_CATALOG_COUNT) # scale the x_points and y_points from the fits pixel coords to the jpg coords fits_height, fits_width = np.shape(sci_hdu.data) x_points, y_points = scale_points(fits_height, fits_width, input['width'], input['height'], x_points=x_points, y_points=y_points) - # we will be giving a list of dicts representing each source back to the frontend + # create the list of source catalog objects source_catalog_data = [] for i in range(SOURCE_CATALOG_COUNT): source_catalog_data.append({ "x": x_points[i], "y": y_points[i], "flux": flux[i].astype(int), - # truncate the ra and dec to 4 decimal places for readability - "ra": '%.4f'%(ra[i]), - "dec": '%.4f'%(dec[i]) + # Astronomical coordinates are formatted to 6 decimal places + "ra": '%.6f'%(ra[i]), + "dec": '%.6f'%(dec[i]) }) return source_catalog_data diff --git a/datalab/datalab_session/views.py b/datalab/datalab_session/views.py index 46d90f9..9147b59 100644 --- a/datalab/datalab_session/views.py +++ b/datalab/datalab_session/views.py @@ -1,3 +1,5 @@ +import logging + from rest_framework.generics import RetrieveAPIView from rest_framework.renderers import JSONRenderer from rest_framework.response import Response @@ -7,6 +9,7 @@ from datalab.datalab_session.analysis.source_catalog import source_catalog from datalab.datalab_session.analysis.get_tif import get_tif from datalab.datalab_session.analysis.raw_data import raw_data +from datalab.datalab_session.exceptions import ClientAlertException class OperationOptionsApiView(RetrieveAPIView): @@ -26,18 +29,32 @@ class AnalysisView(RetrieveAPIView): To add a new analysis action, add a case to the switch statement and create a new file in the analysis directory """ def post(self, request, action): - input = request.data - - match action: - case 'line-profile': - output = line_profile(input) - case 'source-catalog': - output = source_catalog(input) - case 'get-tif': - output = get_tif(input) - case 'raw-data': - output = raw_data(input) - case _: - raise Exception(f'Analysis action {action} not found') - - return Response(output) + log = logging.getLogger() + log.setLevel(logging.INFO) + + try: + # TODO replace switch case with an AnalysisAction class + # AnalysisAction Class Definition + # class AnalysisAction: + # def __init__(self, action): + # def run(self, input): + # def name(self): + input = request.data + + match action: + case 'line-profile': + output = line_profile(input) + case 'source-catalog': + output = source_catalog(input) + case 'get-tif': + output = get_tif(input) + case 'raw-data': + output = raw_data(input) + case _: + raise Exception(f'Analysis action {action} not found') + + return Response(output) + + except ClientAlertException as error: + log.error(f"Error running analysis action {action}: {error}") + return Response({"error": f"Image doesn't support {action}"}, status=400) From 915127d7c8b2ff3c9275d3756575d1483cdcd3e4 Mon Sep 17 00:00:00 2001 From: Lloyd Dakin Date: Mon, 2 Dec 2024 13:57:51 -0800 Subject: [PATCH 2/4] add source to input for analysis, and add 3 decimal places to catalog output --- .../datalab_session/tests/test_analysis.py | 4 +- .../analysis/test_source_catalog.json | 100 +++++++++--------- 2 files changed, 53 insertions(+), 51 deletions(-) diff --git a/datalab/datalab_session/tests/test_analysis.py b/datalab/datalab_session/tests/test_analysis.py index 0c30368..c70c760 100644 --- a/datalab/datalab_session/tests/test_analysis.py +++ b/datalab/datalab_session/tests/test_analysis.py @@ -29,7 +29,8 @@ def test_line_profile(self, mock_get_fits): 'x1': 25, 'y1': 25, 'x2': 75, - 'y2': 75 + 'y2': 75, + 'source': 'archive' }) assert_almost_equal(output.get('line_profile').tolist(), self.test_line_profile_data, decimal=3) @@ -43,6 +44,7 @@ def test_source_catalog(self, mock_get_fits): 'basename': 'fits_1', 'height': 100, 'width': 100, + 'source': 'archive' }) self.assertEqual(output, self.test_source_catalog_data) diff --git a/datalab/datalab_session/tests/test_files/analysis/test_source_catalog.json b/datalab/datalab_session/tests/test_files/analysis/test_source_catalog.json index 775ffab..61651d4 100644 --- a/datalab/datalab_session/tests/test_files/analysis/test_source_catalog.json +++ b/datalab/datalab_session/tests/test_files/analysis/test_source_catalog.json @@ -1,54 +1,54 @@ { "test_source_catalog": [ - {"x": 792, "y": 623, "flux": 3164455, "ra": "260.9333", "dec": "-33.0225"}, - {"x": 919, "y": 120, "flux": 807813, "ra": "260.9667", "dec": "-32.9137"}, - {"x": 996, "y": 627, "flux": 383099, "ra": "260.9861", "dec": "-33.0235"}, - {"x": 997, "y": 641, "flux": 335359, "ra": "260.9862", "dec": "-33.0265"}, - {"x": 587, "y": 803, "flux": 304833, "ra": "260.8802", "dec": "-33.0612"}, - {"x": 1015, "y": 554, "flux": 291847, "ra": "260.9910", "dec": "-33.0076"}, - {"x": 490, "y": 568, "flux": 226494, "ra": "260.8555", "dec": "-33.0101"}, - {"x": 526, "y": 147, "flux": 209307, "ra": "260.8654", "dec": "-32.9191"}, - {"x": 740, "y": 301, "flux": 177543, "ra": "260.9203", "dec": "-32.9526"}, - {"x": 334, "y": 231, "flux": 168666, "ra": "260.8157", "dec": "-32.9370"}, - {"x": 843, "y": 277, "flux": 132858, "ra": "260.9468", "dec": "-32.9476"}, - {"x": 776, "y": 805, "flux": 115180, "ra": "260.9290", "dec": "-33.0618"}, - {"x": 13, "y": 341, "flux": 101317, "ra": "260.7327", "dec": "-32.9604"}, - {"x": 384, "y": 332, "flux": 93054, "ra": "260.8284", "dec": "-32.9590"}, - {"x": 451, "y": 307, "flux": 82848, "ra": "260.8457", "dec": "-32.9536"}, - {"x": 830, "y": 815, "flux": 76200, "ra": "260.9427", "dec": "-33.0641"}, - {"x": 846, "y": 138, "flux": 70788, "ra": "260.9478", "dec": "-32.9176"}, - {"x": 569, "y": 179, "flux": 69761, "ra": "260.8763", "dec": "-32.9261"}, - {"x": 818, "y": 146, "flux": 69329, "ra": "260.9406", "dec": "-32.9191"}, - {"x": 843, "y": 912, "flux": 63748, "ra": "260.9462", "dec": "-33.0851"}, - {"x": 395, "y": 151, "flux": 60681, "ra": "260.8315", "dec": "-32.9198"}, - {"x": 542, "y": 122, "flux": 60653, "ra": "260.8696", "dec": "-32.9136"}, - {"x": 214, "y": 390, "flux": 60057, "ra": "260.7845", "dec": "-32.9712"}, - {"x": 188, "y": 691, "flux": 59835, "ra": "260.7774", "dec": "-33.0363"}, - {"x": 173, "y": 438, "flux": 59768, "ra": "260.7738", "dec": "-32.9816"}, - {"x": 1022, "y": 188, "flux": 56811, "ra": "260.9931", "dec": "-32.9285"}, - {"x": 431, "y": 936, "flux": 56575, "ra": "260.8396", "dec": "-33.0898"}, - {"x": 333, "y": 112, "flux": 51136, "ra": "260.8157", "dec": "-32.9112"}, - {"x": 211, "y": 958, "flux": 50722, "ra": "260.7828", "dec": "-33.0941"}, - {"x": 664, "y": 535, "flux": 45352, "ra": "260.9004", "dec": "-33.0033"}, - {"x": 989, "y": 975, "flux": 44414, "ra": "260.9838", "dec": "-33.0988"}, - {"x": 1003, "y": 463, "flux": 38238, "ra": "260.9878", "dec": "-32.9879"}, - {"x": 336, "y": 242, "flux": 37970, "ra": "260.8161", "dec": "-32.9393"}, - {"x": 294, "y": 565, "flux": 37707, "ra": "260.8050", "dec": "-33.0093"}, - {"x": 535, "y": 916, "flux": 37575, "ra": "260.8666", "dec": "-33.0854"}, - {"x": 66, "y": 415, "flux": 37365, "ra": "260.7463", "dec": "-32.9765"}, - {"x": 533, "y": 91, "flux": 36539, "ra": "260.8673", "dec": "-32.9069"}, - {"x": 879, "y": 505, "flux": 35138, "ra": "260.9559", "dec": "-32.9970"}, - {"x": 293, "y": 638, "flux": 34703, "ra": "260.8045", "dec": "-33.0250"}, - {"x": 444, "y": 635, "flux": 33406, "ra": "260.8434", "dec": "-33.0245"}, - {"x": 400, "y": 82, "flux": 33366, "ra": "260.8330", "dec": "-32.9048"}, - {"x": 465, "y": 59, "flux": 32920, "ra": "260.8497", "dec": "-32.8998"}, - {"x": 874, "y": 50, "flux": 32280, "ra": "260.9553", "dec": "-32.8984"}, - {"x": 1007, "y": 416, "flux": 29422, "ra": "260.9892", "dec": "-32.9779"}, - {"x": 281, "y": 294, "flux": 28626, "ra": "260.8019", "dec": "-32.9506"}, - {"x": 682, "y": 39, "flux": 28508, "ra": "260.9057", "dec": "-32.8959"}, - {"x": 972, "y": 426, "flux": 27457, "ra": "260.9801", "dec": "-32.9799"}, - {"x": 480, "y": 557, "flux": 26744, "ra": "260.8529", "dec": "-33.0077"}, - {"x": 812, "y": 291, "flux": 25618, "ra": "260.9389", "dec": "-32.9506"}, - {"x": 443, "y": 206, "flux": 25483, "ra": "260.8438", "dec": "-32.9317"} + {"x": 792, "y": 623, "flux": 3164455, "ra": "260.933350", "dec": "-33.022468"}, + {"x": 919, "y": 120, "flux": 807813, "ra": "260.966724", "dec": "-32.913659"}, + {"x": 996, "y": 627, "flux": 383099, "ra": "260.986084", "dec": "-33.023544"}, + {"x": 997, "y": 641, "flux": 335359, "ra": "260.986153", "dec": "-33.026504"}, + {"x": 587, "y": 803, "flux": 304833, "ra": "260.880190", "dec": "-33.061203"}, + {"x": 1015, "y": 554, "flux": 291847, "ra": "260.990955", "dec": "-33.007606"}, + {"x": 490, "y": 568, "flux": 226494, "ra": "260.855531", "dec": "-33.010065"}, + {"x": 526, "y": 147, "flux": 209307, "ra": "260.865447", "dec": "-32.919082"}, + {"x": 740, "y": 301, "flux": 177543, "ra": "260.920254", "dec": "-32.952604"}, + {"x": 334, "y": 231, "flux": 168666, "ra": "260.815714", "dec": "-32.937012"}, + {"x": 843, "y": 277, "flux": 132858, "ra": "260.946803", "dec": "-32.947639"}, + {"x": 776, "y": 805, "flux": 115180, "ra": "260.929046", "dec": "-33.061751"}, + {"x": 13, "y": 341, "flux": 101317, "ra": "260.732683", "dec": "-32.960356"}, + {"x": 384, "y": 332, "flux": 93054, "ra": "260.828434", "dec": "-32.958980"}, + {"x": 451, "y": 307, "flux": 82848, "ra": "260.845680", "dec": "-32.953588"}, + {"x": 830, "y": 815, "flux": 76200, "ra": "260.942738", "dec": "-33.064078"}, + {"x": 846, "y": 138, "flux": 70788, "ra": "260.947787", "dec": "-32.917576"}, + {"x": 569, "y": 179, "flux": 69761, "ra": "260.876290", "dec": "-32.926068"}, + {"x": 818, "y": 146, "flux": 69329, "ra": "260.940558", "dec": "-32.919116"}, + {"x": 843, "y": 912, "flux": 63748, "ra": "260.946216", "dec": "-33.085093"}, + {"x": 395, "y": 151, "flux": 60681, "ra": "260.831455", "dec": "-32.919758"}, + {"x": 542, "y": 122, "flux": 60653, "ra": "260.869564", "dec": "-32.913619"}, + {"x": 214, "y": 390, "flux": 60057, "ra": "260.784459", "dec": "-32.971195"}, + {"x": 188, "y": 691, "flux": 59835, "ra": "260.777363", "dec": "-33.036278"}, + {"x": 173, "y": 438, "flux": 59768, "ra": "260.773825", "dec": "-32.981611"}, + {"x": 1022, "y": 188, "flux": 56811, "ra": "260.993067", "dec": "-32.928477"}, + {"x": 431, "y": 936, "flux": 56575, "ra": "260.839617", "dec": "-33.089795"}, + {"x": 333, "y": 112, "flux": 51136, "ra": "260.815677", "dec": "-32.911207"}, + {"x": 211, "y": 958, "flux": 50722, "ra": "260.782797", "dec": "-33.094140"}, + {"x": 664, "y": 535, "flux": 45352, "ra": "260.900375", "dec": "-33.003295"}, + {"x": 989, "y": 975, "flux": 44414, "ra": "260.983777", "dec": "-33.098763"}, + {"x": 1003, "y": 463, "flux": 38238, "ra": "260.987847", "dec": "-32.987939"}, + {"x": 336, "y": 242, "flux": 37970, "ra": "260.816099", "dec": "-32.939259"}, + {"x": 294, "y": 565, "flux": 37707, "ra": "260.804958", "dec": "-33.009319"}, + {"x": 535, "y": 916, "flux": 37575, "ra": "260.866625", "dec": "-33.085422"}, + {"x": 66, "y": 415, "flux": 37365, "ra": "260.746322", "dec": "-32.976492"}, + {"x": 533, "y": 91, "flux": 36539, "ra": "260.867257", "dec": "-32.906923"}, + {"x": 879, "y": 505, "flux": 35138, "ra": "260.955856", "dec": "-32.996953"}, + {"x": 293, "y": 638, "flux": 34703, "ra": "260.804463", "dec": "-33.024956"}, + {"x": 444, "y": 635, "flux": 33406, "ra": "260.843397", "dec": "-33.024536"}, + {"x": 400, "y": 82, "flux": 33366, "ra": "260.832988", "dec": "-32.904823"}, + {"x": 465, "y": 59, "flux": 32920, "ra": "260.849739", "dec": "-32.899831"}, + {"x": 874, "y": 50, "flux": 32280, "ra": "260.955293", "dec": "-32.898351"}, + {"x": 1007, "y": 416, "flux": 29422, "ra": "260.989158", "dec": "-32.977875"}, + {"x": 281, "y": 294, "flux": 28626, "ra": "260.801870", "dec": "-32.950643"}, + {"x": 682, "y": 39, "flux": 28508, "ra": "260.905706", "dec": "-32.895904"}, + {"x": 972, "y": 426, "flux": 27457, "ra": "260.980081", "dec": "-32.979940"}, + {"x": 480, "y": 557, "flux": 26744, "ra": "260.852921", "dec": "-33.007667"}, + {"x": 812, "y": 291, "flux": 25618, "ra": "260.938918", "dec": "-32.950600"}, + {"x": 443, "y": 206, "flux": 25483, "ra": "260.843795", "dec": "-32.931662"} ] } From c86fdf50fd88c2081ca3a9722b64d8390cc126dc Mon Sep 17 00:00:00 2001 From: Lloyd Dakin Date: Tue, 3 Dec 2024 10:51:18 -0800 Subject: [PATCH 3/4] not attaching ra and dec if null --- .../datalab_session/analysis/source_catalog.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/datalab/datalab_session/analysis/source_catalog.py b/datalab/datalab_session/analysis/source_catalog.py index e856512..508ed40 100644 --- a/datalab/datalab_session/analysis/source_catalog.py +++ b/datalab/datalab_session/analysis/source_catalog.py @@ -43,8 +43,8 @@ def source_catalog(input: dict): dec = cat_hdu.data["dec"][:SOURCE_CATALOG_COUNT] else: # TODO: implement a fallback way to calculate ra, dec from x, y and WCS - ra = np.zeros(SOURCE_CATALOG_COUNT) - dec = np.zeros(SOURCE_CATALOG_COUNT) + ra = None + dec = None # scale the x_points and y_points from the fits pixel coords to the jpg coords fits_height, fits_width = np.shape(sci_hdu.data) @@ -53,13 +53,15 @@ def source_catalog(input: dict): # create the list of source catalog objects source_catalog_data = [] for i in range(SOURCE_CATALOG_COUNT): - source_catalog_data.append({ + source_data = { "x": x_points[i], "y": y_points[i], - "flux": flux[i].astype(int), - # Astronomical coordinates are formatted to 6 decimal places - "ra": '%.6f'%(ra[i]), - "dec": '%.6f'%(dec[i]) - }) + "flux": flux[i].astype(int) + } + if ra and dec: + source_data["ra"] = '%.6f' % (ra[i]) + source_data["dec"] = '%.6f' % (dec[i]) + + source_catalog_data.append(source_data) return source_catalog_data From f72db73fc9650e76d55c2218845db7aade0c4640 Mon Sep 17 00:00:00 2001 From: Lloyd Dakin Date: Tue, 3 Dec 2024 10:54:26 -0800 Subject: [PATCH 4/4] change check to be is not None to satisfy tests --- datalab/datalab_session/analysis/source_catalog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datalab/datalab_session/analysis/source_catalog.py b/datalab/datalab_session/analysis/source_catalog.py index 508ed40..049cdc0 100644 --- a/datalab/datalab_session/analysis/source_catalog.py +++ b/datalab/datalab_session/analysis/source_catalog.py @@ -58,7 +58,7 @@ def source_catalog(input: dict): "y": y_points[i], "flux": flux[i].astype(int) } - if ra and dec: + if ra is not None and dec is not None: source_data["ra"] = '%.6f' % (ra[i]) source_data["dec"] = '%.6f' % (dec[i])