From 566dbc478c70737b59182afeb06bb16c997c95c5 Mon Sep 17 00:00:00 2001 From: Lloyd Dakin Date: Wed, 1 May 2024 15:56:35 -0700 Subject: [PATCH 1/4] analysis endpoing, hdu util, and line_profile --- .../datalab_session/analysis/line_profile.py | 34 +++++++++++++++++++ datalab/datalab_session/util.py | 30 ++++++++++++++++ datalab/datalab_session/views.py | 21 ++++++++++++ datalab/urls.py | 3 +- 4 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 datalab/datalab_session/analysis/line_profile.py diff --git a/datalab/datalab_session/analysis/line_profile.py b/datalab/datalab_session/analysis/line_profile.py new file mode 100644 index 0000000..f78d449 --- /dev/null +++ b/datalab/datalab_session/analysis/line_profile.py @@ -0,0 +1,34 @@ +import numpy as np +from skimage.measure import profile_line + +# For creating an array of brightness along a user drawn line +def line_profile(input, sci_hdu): + """ + Given a set of coordinates, return the line profile of the image + """ + points = scale_points(input['width'], input['height'], sci_hdu.data, [(input["x1"], input["y1"]), (input["x2"], input["y2"])]) + line_profile = profile_line(sci_hdu.data, points[0], points[1], mode="constant", cval=-1) + arcsec = len(line_profile) * sci_hdu.header["PIXSCALE"] + + return {"line_profile": line_profile, "arcsec": arcsec} + + +def scale_points(small_img_width, small_img_height, img_array, points: list[tuple[int, int]]): + """ + Scale the coordinates from a smaller image to the full sized fits so we know the positions of the coords on the 2dnumpy array + Returns the list of tuple points with coords scaled for the numpy array + """ + + large_height, large_width = np.shape(img_array) + + # If the aspect ratios don't match we can't be certain where the point was + if small_img_width / small_img_height != large_width / large_height: + raise ValueError("Aspect ratios of the two images must match") + + width_scale = large_width / small_img_width + height_scale = large_height / small_img_height + + points_array = np.array(points) + scaled_points = np.int_(points_array * [width_scale, height_scale]) + + return scaled_points diff --git a/datalab/datalab_session/util.py b/datalab/datalab_session/util.py index 91ce4b5..b6fe38f 100644 --- a/datalab/datalab_session/util.py +++ b/datalab/datalab_session/util.py @@ -97,6 +97,36 @@ def get_archive_from_basename(basename: str) -> dict: return results +def get_hdus(basenames) -> list[fits.HDUList]: + """ + Returns a list of HDULists for the given basenames + Warning: this function returns an opened file that must be closed after use + """ + if isinstance(basenames, str): + basenames = [basenames] + + hdu_list = [] + + for basename in basenames: + basename = basename.replace('-large', '').replace('-small', '') + + archive_record = get_archive_from_basename(basename) + + try: + fits_url = archive_record[0].get('url', 'No URL found') + except IndexError: + RuntimeWarning(f"No image found with specified basename: {basename}") + continue + + hdu = fits.open(fits_url) + + hdu_list.append(hdu) + + if len(hdu_list) == 1: + return hdu_list[0] + else: + return hdu_list + def create_fits(key: str, image_arr: np.ndarray) -> fits.HDUList: header = fits.Header([('KEY', key)]) diff --git a/datalab/datalab_session/views.py b/datalab/datalab_session/views.py index a59f33c..1231457 100644 --- a/datalab/datalab_session/views.py +++ b/datalab/datalab_session/views.py @@ -1,8 +1,11 @@ from rest_framework.generics import RetrieveAPIView +from rest_framework.views import APIView from rest_framework.renderers import JSONRenderer from rest_framework.response import Response from datalab.datalab_session.data_operations.utils import available_operations +from datalab.datalab_session.analysis.line_profile import line_profile +from datalab.datalab_session.util import get_hdus class OperationOptionsApiView(RetrieveAPIView): @@ -15,3 +18,21 @@ def get(self, request): for operation_clazz in operations.values(): operation_details[operation_clazz.name()] = operation_clazz.wizard_description() return Response(operation_details) + +class AnalysisView(APIView): + """ View to handle analysis actions and return the results """ + def post(self, request, action): + input = request.data + + hdu = get_hdus(input['basename']) + sci = hdu['SCI'] + + match action: + case 'line-profile': + output = line_profile(input, sci) + case _: + raise Exception(f'Analysis action {action} not found') + + hdu.close() + + return Response(output) diff --git a/datalab/urls.py b/datalab/urls.py index 9cb0ff7..234108b 100644 --- a/datalab/urls.py +++ b/datalab/urls.py @@ -20,7 +20,7 @@ import ocs_authentication.auth_profile.urls as authprofile_urls from datalab.datalab_session.viewsets import DataSessionViewSet, DataOperationViewSet -from datalab.datalab_session.views import OperationOptionsApiView +from datalab.datalab_session.views import OperationOptionsApiView, AnalysisView router = routers.SimpleRouter() router.register(r'datasessions', DataSessionViewSet, 'datasessions') @@ -35,6 +35,7 @@ urlpatterns = [ path('admin/', admin.site.urls), re_path(r'^api/', include(api_urlpatterns)), + path(r'api/analysis//', AnalysisView.as_view(), name='analysis'), path('api/available_operations/', OperationOptionsApiView.as_view(), name='available_operations'), re_path(r'^authprofile/', include(authprofile_urls)), ] From 5d2e27685bb3ed5504e4cd379fcb51fe6f6f019d Mon Sep 17 00:00:00 2001 From: Lloyd Dakin Date: Thu, 2 May 2024 10:21:01 -0700 Subject: [PATCH 2/4] caching and comments --- datalab/datalab_session/analysis/line_profile.py | 2 +- datalab/datalab_session/util.py | 9 ++++++++- datalab/datalab_session/views.py | 5 ++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/datalab/datalab_session/analysis/line_profile.py b/datalab/datalab_session/analysis/line_profile.py index f78d449..e714cbd 100644 --- a/datalab/datalab_session/analysis/line_profile.py +++ b/datalab/datalab_session/analysis/line_profile.py @@ -4,7 +4,7 @@ # For creating an array of brightness along a user drawn line def line_profile(input, sci_hdu): """ - Given a set of coordinates, return the line profile of the image + Creates an array of luminosity values and the length of the line in arcseconds """ points = scale_points(input['width'], input['height'], sci_hdu.data, [(input["x1"], input["y1"]), (input["x2"], input["y2"])]) line_profile = profile_line(sci_hdu.data, points[0], points[1], mode="constant", cval=-1) diff --git a/datalab/datalab_session/util.py b/datalab/datalab_session/util.py index b6fe38f..f1b71a2 100644 --- a/datalab/datalab_session/util.py +++ b/datalab/datalab_session/util.py @@ -6,6 +6,7 @@ import numpy as np from django.conf import settings +from django.core.cache import cache log = logging.getLogger() log.setLevel(logging.INFO) @@ -107,9 +108,14 @@ def get_hdus(basenames) -> list[fits.HDUList]: hdu_list = [] + # use the basename to fetch and create a list of hdu objects for basename in basenames: basename = basename.replace('-large', '').replace('-small', '') + if cache.get(basename) is not None: + hdu_list.append(cache.get(basename)) + continue + archive_record = get_archive_from_basename(basename) try: @@ -118,7 +124,8 @@ def get_hdus(basenames) -> list[fits.HDUList]: RuntimeWarning(f"No image found with specified basename: {basename}") continue - hdu = fits.open(fits_url) + hdu = fits.open(fits_url, use_fsspec=True) + cache.set(basename, hdu) hdu_list.append(hdu) diff --git a/datalab/datalab_session/views.py b/datalab/datalab_session/views.py index 1231457..9d0f260 100644 --- a/datalab/datalab_session/views.py +++ b/datalab/datalab_session/views.py @@ -20,7 +20,10 @@ def get(self, request): return Response(operation_details) class AnalysisView(APIView): - """ View to handle analysis actions and return the results """ + """ + View to handle analysis actions and return the results + To add a new analysis action, add a case to the switch statement and create a new file in the analysis directory + """ def post(self, request, action): input = request.data From f3eeb0d6a75315866f6f1c91aeef679318b1b5a9 Mon Sep 17 00:00:00 2001 From: Lloyd Dakin Date: Mon, 6 May 2024 12:07:19 -0700 Subject: [PATCH 3/4] get_hdu now has an optional extention arguement default to SCI, works with just one basename instead of lists, moved scale_points to util file --- .../datalab_session/analysis/line_profile.py | 26 +------- datalab/datalab_session/util.py | 60 +++++++++++-------- datalab/datalab_session/views.py | 9 +-- 3 files changed, 40 insertions(+), 55 deletions(-) diff --git a/datalab/datalab_session/analysis/line_profile.py b/datalab/datalab_session/analysis/line_profile.py index e714cbd..d3382f7 100644 --- a/datalab/datalab_session/analysis/line_profile.py +++ b/datalab/datalab_session/analysis/line_profile.py @@ -1,8 +1,9 @@ -import numpy as np from skimage.measure import profile_line +from datalab.datalab_session.util import scale_points + # For creating an array of brightness along a user drawn line -def line_profile(input, sci_hdu): +def line_profile(input: dict, sci_hdu: object): """ Creates an array of luminosity values and the length of the line in arcseconds """ @@ -11,24 +12,3 @@ def line_profile(input, sci_hdu): arcsec = len(line_profile) * sci_hdu.header["PIXSCALE"] return {"line_profile": line_profile, "arcsec": arcsec} - - -def scale_points(small_img_width, small_img_height, img_array, points: list[tuple[int, int]]): - """ - Scale the coordinates from a smaller image to the full sized fits so we know the positions of the coords on the 2dnumpy array - Returns the list of tuple points with coords scaled for the numpy array - """ - - large_height, large_width = np.shape(img_array) - - # If the aspect ratios don't match we can't be certain where the point was - if small_img_width / small_img_height != large_width / large_height: - raise ValueError("Aspect ratios of the two images must match") - - width_scale = large_width / small_img_width - height_scale = large_height / small_img_height - - points_array = np.array(points) - scaled_points = np.int_(points_array * [width_scale, height_scale]) - - return scaled_points diff --git a/datalab/datalab_session/util.py b/datalab/datalab_session/util.py index f1b71a2..6aebf98 100644 --- a/datalab/datalab_session/util.py +++ b/datalab/datalab_session/util.py @@ -98,41 +98,29 @@ def get_archive_from_basename(basename: str) -> dict: return results -def get_hdus(basenames) -> list[fits.HDUList]: +def get_hdu(basename: str, extension: str = 'SCI') -> list[fits.HDUList]: """ - Returns a list of HDULists for the given basenames + Returns a list of Sci HDUs for the given basenames Warning: this function returns an opened file that must be closed after use """ - if isinstance(basenames, str): - basenames = [basenames] - - hdu_list = [] # use the basename to fetch and create a list of hdu objects - for basename in basenames: - basename = basename.replace('-large', '').replace('-small', '') - - if cache.get(basename) is not None: - hdu_list.append(cache.get(basename)) - continue + basename = basename.replace('-large', '').replace('-small', '') - archive_record = get_archive_from_basename(basename) + if cache.get(f'{basename}-{extension}') is not None: + return cache.get(f'{basename}-{extension}') - try: - fits_url = archive_record[0].get('url', 'No URL found') - except IndexError: - RuntimeWarning(f"No image found with specified basename: {basename}") - continue + archive_record = get_archive_from_basename(basename) - hdu = fits.open(fits_url, use_fsspec=True) - cache.set(basename, hdu) - - hdu_list.append(hdu) + try: + fits_url = archive_record[0].get('url', 'No URL found') + except IndexError: + RuntimeWarning(f"No image found with specified basename: {basename}") - if len(hdu_list) == 1: - return hdu_list[0] - else: - return hdu_list + # not sure if HDU is ever being closed here? + hdu = fits.open(fits_url, use_fsspec=True) + cache.set(f'{basename}-{extension}', hdu[extension]) + return hdu[extension] def create_fits(key: str, image_arr: np.ndarray) -> fits.HDUList: @@ -155,3 +143,23 @@ def stack_arrays(array_list: list): stacked = np.stack(cropped_data_list, axis=2) return stacked + +def scale_points(small_img_width: int, small_img_height: int, img_array: list, points: list[tuple[int, int]]): + """ + Scale the coordinates from a smaller image to the full sized fits so we know the positions of the coords on the 2dnumpy array + Returns the list of tuple points with coords scaled for the numpy array + """ + + large_height, large_width = np.shape(img_array) + + # If the aspect ratios don't match we can't be certain where the point was + if small_img_width / small_img_height != large_width / large_height: + raise ValueError("Aspect ratios of the two images must match") + + width_scale = large_width / small_img_width + height_scale = large_height / small_img_height + + points_array = np.array(points) + scaled_points = np.int_(points_array * [width_scale, height_scale]) + + return scaled_points diff --git a/datalab/datalab_session/views.py b/datalab/datalab_session/views.py index 9d0f260..40d1994 100644 --- a/datalab/datalab_session/views.py +++ b/datalab/datalab_session/views.py @@ -5,7 +5,7 @@ from datalab.datalab_session.data_operations.utils import available_operations from datalab.datalab_session.analysis.line_profile import line_profile -from datalab.datalab_session.util import get_hdus +from datalab.datalab_session.util import get_hdu class OperationOptionsApiView(RetrieveAPIView): @@ -27,15 +27,12 @@ class AnalysisView(APIView): def post(self, request, action): input = request.data - hdu = get_hdus(input['basename']) - sci = hdu['SCI'] + sci_hdu = get_hdu(input['basename']) match action: case 'line-profile': - output = line_profile(input, sci) + output = line_profile(input, sci_hdu) case _: raise Exception(f'Analysis action {action} not found') - - hdu.close() return Response(output) From 52459a9fba8352fd1243658a7e1b76ce1e2f14bb Mon Sep 17 00:00:00 2001 From: Lloyd Dakin Date: Mon, 6 May 2024 15:46:11 -0700 Subject: [PATCH 4/4] removed comment --- datalab/datalab_session/util.py | 1 - 1 file changed, 1 deletion(-) diff --git a/datalab/datalab_session/util.py b/datalab/datalab_session/util.py index 6aebf98..19ab1a9 100644 --- a/datalab/datalab_session/util.py +++ b/datalab/datalab_session/util.py @@ -117,7 +117,6 @@ def get_hdu(basename: str, extension: str = 'SCI') -> list[fits.HDUList]: except IndexError: RuntimeWarning(f"No image found with specified basename: {basename}") - # not sure if HDU is ever being closed here? hdu = fits.open(fits_url, use_fsspec=True) cache.set(f'{basename}-{extension}', hdu[extension]) return hdu[extension]