From 33d4cd7e3e0c2fdf87bacfc29ee8ac1b7cb17458 Mon Sep 17 00:00:00 2001 From: Lloyd Dakin Date: Mon, 29 Jul 2024 16:26:10 -0700 Subject: [PATCH 1/5] rgb operation, spelling maximum fix, util function changes --- .../data_operations/data_operation.py | 70 +++++++++------- .../datalab_session/data_operations/long.py | 2 +- .../datalab_session/data_operations/median.py | 6 +- .../datalab_session/data_operations/noop.py | 2 +- .../data_operations/rgb_stack.py | 82 +++++++++++++++++++ datalab/datalab_session/util.py | 33 ++++++-- 6 files changed, 154 insertions(+), 41 deletions(-) create mode 100644 datalab/datalab_session/data_operations/rgb_stack.py diff --git a/datalab/datalab_session/data_operations/data_operation.py b/datalab/datalab_session/data_operations/data_operation.py index 40edd6a..b6a22e8 100644 --- a/datalab/datalab_session/data_operations/data_operation.py +++ b/datalab/datalab_session/data_operations/data_operation.py @@ -9,7 +9,7 @@ import numpy as np from datalab.datalab_session.tasks import execute_data_operation -from datalab.datalab_session.util import add_file_to_bucket, get_hdu +from datalab.datalab_session.util import add_file_to_bucket, get_hdu, fits_dimensions, stack_arrays, create_fits CACHE_DURATION = 60 * 60 * 24 * 30 # cache for 30 days @@ -99,42 +99,54 @@ def set_failed(self, message: str): self.set_status('FAILED') self.set_message(message) - # percent lets you allocate a fraction of the operation that this takes up in time - # cur_percent is the current completion of the operation - def create_and_store_fits(self, hdu_list: fits.HDUList, percent=None, cur_percent=None) -> list: - if not type(hdu_list) == list: - hdu_list = [hdu_list] + def create_jpg_output(self, fits_paths, percent=None, cur_percent=None, color=False, index=None) -> list: + """ + Create jpgs from fits files and save them to S3 + If using the color option fits_paths need to be in order R, G, B + percent and cur_percent are used to update the progress of the operation + """ - output = [] - total_files = len(hdu_list) + if type(fits_paths) is not list: + fits_paths = [fits_paths] - # Create temp file paths for storing the products - fits_path = tempfile.NamedTemporaryFile(suffix=f'{self.cache_key}.fits').name + # create the jpgs from the fits files large_jpg_path = tempfile.NamedTemporaryFile(suffix=f'{self.cache_key}-large.jpg').name thumbnail_jpg_path = tempfile.NamedTemporaryFile(suffix=f'{self.cache_key}-small.jpg').name - for index, hdu in enumerate(hdu_list, start=1): - height, width = hdu[1].shape + height, width = fits_dimensions(fits_paths[0]) - hdu.writeto(fits_path) - fits_to_jpg(fits_path, large_jpg_path, width=width, height=height) - fits_to_jpg(fits_path, thumbnail_jpg_path) + fits_to_jpg(fits_paths, large_jpg_path, width=width, height=height, color=color) + fits_to_jpg(fits_paths, thumbnail_jpg_path, color=color) - # Save Fits and Thumbnails in S3 Buckets - fits_url = add_file_to_bucket(f'{self.cache_key}/{self.cache_key}-{index}.fits', fits_path) - large_jpg_url = add_file_to_bucket(f'{self.cache_key}/{self.cache_key}-{index}-large.jpg', large_jpg_path) - thumbnail_jpg_url = add_file_to_bucket(f'{self.cache_key}/{self.cache_key}-{index}-small.jpg', thumbnail_jpg_path) - - output.append({ - 'fits_url': fits_url, - 'large_url': large_jpg_url, - 'thumbnail_url': thumbnail_jpg_url, - 'basename': f'{self.cache_key}-{index}', - 'source': 'datalab'} - ) + # color photos take three files, so we store it as one fits file with a 3d SCI ndarray + if color: + arrays = [fits.open(file)['SCI'].data for file in fits_paths] + stacked = stack_arrays(arrays) + fits_file = create_fits(self.cache_key, stacked) + else: + fits_file = fits_paths[0] - if percent is not None and cur_percent is not None: - self.set_percent_completion(cur_percent + index/total_files * percent) + + # Save Fits and Thumbnails in S3 Buckets + bucket_key = f'{self.cache_key}/{self.cache_key}-{index}' if index else f'{self.cache_key}/{self.cache_key}' + + fits_url = add_file_to_bucket(f'{bucket_key}.fits', fits_file) + large_jpg_url = add_file_to_bucket(f'{bucket_key}-large.jpg', large_jpg_path) + thumbnail_jpg_url = add_file_to_bucket(f'{bucket_key}-small.jpg', thumbnail_jpg_path) + + output = [] + output.append({ + 'fits_url': fits_url, + 'large_url': large_jpg_url, + 'thumbnail_url': thumbnail_jpg_url, + 'basename': f'{self.cache_key}', + 'source': 'datalab'} + ) + + if percent is not None and cur_percent is not None: + self.set_percent_completion(cur_percent + percent) + else: + self.set_percent_completion(0.9) return output diff --git a/datalab/datalab_session/data_operations/long.py b/datalab/datalab_session/data_operations/long.py index 4146adc..78c889f 100644 --- a/datalab/datalab_session/data_operations/long.py +++ b/datalab/datalab_session/data_operations/long.py @@ -23,7 +23,7 @@ def wizard_description(): 'description': 'The input files to operate on', 'type': 'file', 'minimum': 1, - 'maxmimum': 999 + 'maximum': 999 }, 'duration': { 'name': 'Duration', diff --git a/datalab/datalab_session/data_operations/median.py b/datalab/datalab_session/data_operations/median.py index 76693db..55e94fd 100644 --- a/datalab/datalab_session/data_operations/median.py +++ b/datalab/datalab_session/data_operations/median.py @@ -33,7 +33,7 @@ def wizard_description(): 'description': 'The input files to operate on', 'type': 'file', 'minimum': 1, - 'maxmimum': 999 + 'maximum': 999 } } } @@ -52,9 +52,9 @@ def operate(self): # using the numpy library's median method median = np.median(stacked_data, axis=2) - hdu_list = create_fits(self.cache_key, median) + fits_file = create_fits(self.cache_key, median) - output = self.create_and_store_fits(hdu_list, percent=0.6, cur_percent=0.4) + output = self.create_jpg_output(fits_file, percent=0.6, cur_percent=0.4) output = {'output_files': output} else: diff --git a/datalab/datalab_session/data_operations/noop.py b/datalab/datalab_session/data_operations/noop.py index 9905ef4..35931be 100644 --- a/datalab/datalab_session/data_operations/noop.py +++ b/datalab/datalab_session/data_operations/noop.py @@ -22,7 +22,7 @@ def wizard_description(): 'description': 'The input files to operate on', 'type': 'file', 'minimum': 1, - 'maxmimum': 999 + 'maximum': 999 }, 'scalar_parameter_1': { 'name': 'Scalar Parameter 1', diff --git a/datalab/datalab_session/data_operations/rgb_stack.py b/datalab/datalab_session/data_operations/rgb_stack.py new file mode 100644 index 0000000..d5a5dbe --- /dev/null +++ b/datalab/datalab_session/data_operations/rgb_stack.py @@ -0,0 +1,82 @@ +import logging +import tempfile + +from fits2image.conversions import fits_to_jpg + +from datalab.datalab_session.data_operations.data_operation import BaseDataOperation +from datalab.datalab_session.util import add_file_to_bucket, get_fits + +log = logging.getLogger() +log.setLevel(logging.INFO) + + +class RGB_Stack(BaseDataOperation): + + @staticmethod + def name(): + return 'RGB Stack' + + @staticmethod + def description(): + return """The RGB Stack operation takes in 3 input images which have red, green, and blue filters and creates a colored image by compositing them on top of each other.""" + + @staticmethod + def wizard_description(): + return { + 'name': RGB_Stack.name(), + 'description': RGB_Stack.description(), + 'category': 'image', + 'inputs': { + 'red_input': { + 'name': 'Red Filter', + 'description': 'Three images to stack their RGB values', + 'type': 'file', + 'minimum': 1, + 'maximum': 1, + 'filter': ['rp', 'r'] + }, + 'green_input': { + 'name': 'Green Filter', + 'description': 'Three images to stack their RGB values', + 'type': 'file', + 'minimum': 1, + 'maximum': 1, + 'filter': ['V'] + }, + 'blue_input': { + 'name': 'Blue Filter', + 'description': 'Three images to stack their RGB values', + 'type': 'file', + 'minimum': 1, + 'maximum': 1, + 'filter': ['B'] + }, + 'random_seed': { + 'name': 'Seed', + 'description': 'so I can test this without it being cached', + 'type': 'number', + 'minimum': 0, + 'maximum': 99999.0, + 'default': 60.0 + } + } + } + + def operate(self): + rgb_input_list = self.input_data['red_input'] + self.input_data['green_input'] + self.input_data['blue_input'] + + if len(rgb_input_list) == 3: + log.info(f'Executing RGB Stack operation on files: {rgb_input_list}') + + fits_paths = [] + for file in rgb_input_list: + fits_paths.append(get_fits(file.get('basename'))) + + output = self.create_jpg_output(fits_paths, percent=0.9, cur_percent=0.0, color=True) + else: + output = {'output_files': []} + raise ValueError('RGB Stack operation requires exactly 3 input files') + + self.set_percent_completion(1.0) + self.set_output(output) + log.info(f'RGB Stack output: {self.get_output()}') diff --git a/datalab/datalab_session/util.py b/datalab/datalab_session/util.py index 9d6446b..0add14a 100644 --- a/datalab/datalab_session/util.py +++ b/datalab/datalab_session/util.py @@ -1,3 +1,4 @@ +import tempfile import requests import logging import os @@ -111,14 +112,10 @@ def get_archive_url(basename: str, archive: str = settings.ARCHIVE_API) -> dict: fits_url = results[0].get('url', 'No URL found') return fits_url -def get_hdu(basename: str, extension: str = 'SCI', source: str = 'archive') -> list[fits.HDUList]: +def get_fits(basename: str, source: str = 'archive'): """ - Returns a HDU for the given basename from the source - Will download the file to a tmp directory so future calls can open it directly - Warning: this function returns an opened file that must be closed after use + Returns a Fits File for the given basename from the source """ - - # use the basename to fetch and create a list of hdu objects basename = basename.replace('-large', '').replace('-small', '') basename_file_path = os.path.join(settings.TEMP_FITS_DIR, basename) @@ -139,6 +136,17 @@ def get_hdu(basename: str, extension: str = 'SCI', source: str = 'archive') -> l raise ValueError(f"Source {source} not recognized") urllib.request.urlretrieve(fits_url, basename_file_path) + + return basename_file_path + +def get_hdu(basename: str, extension: str = 'SCI', source: str = 'archive') -> list[fits.HDUList]: + """ + Returns a HDU for the given basename from the source + Will download the file to a tmp directory so future calls can open it directly + Warning: this function returns an opened file that must be closed after use + """ + + basename_file_path = get_fits(basename, source) hdu = fits.open(basename_file_path) try: @@ -148,15 +156,26 @@ def get_hdu(basename: str, extension: str = 'SCI', source: str = 'archive') -> l return extension +def fits_dimensions(fits_file) -> tuple: + hdu = fits.open(fits_file) + height, width = hdu[1].shape + return height, width + def create_fits(key: str, image_arr: np.ndarray) -> fits.HDUList: + """ + Creates a fits file with the given key and image array + Returns the HDUList and the dimensions of the image + """ header = fits.Header([('KEY', key)]) primary_hdu = fits.PrimaryHDU(header=header) image_hdu = fits.ImageHDU(data=image_arr, name='SCI') hdu_list = fits.HDUList([primary_hdu, image_hdu]) + fits_path = tempfile.NamedTemporaryFile(suffix=f'{key}.fits').name + hdu_list.writeto(fits_path) - return hdu_list + return fits_path def stack_arrays(array_list: list): """ From 3c9cd064ad9b1bc8c9f18b9b61042a39cb42f613 Mon Sep 17 00:00:00 2001 From: Lloyd Dakin Date: Tue, 30 Jul 2024 11:19:53 -0700 Subject: [PATCH 2/5] add gp filter, set output correctly --- datalab/datalab_session/data_operations/rgb_stack.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/datalab/datalab_session/data_operations/rgb_stack.py b/datalab/datalab_session/data_operations/rgb_stack.py index d5a5dbe..4bea235 100644 --- a/datalab/datalab_session/data_operations/rgb_stack.py +++ b/datalab/datalab_session/data_operations/rgb_stack.py @@ -41,7 +41,7 @@ def wizard_description(): 'type': 'file', 'minimum': 1, 'maximum': 1, - 'filter': ['V'] + 'filter': ['V', 'gp'] }, 'blue_input': { 'name': 'Blue Filter', @@ -73,6 +73,8 @@ def operate(self): fits_paths.append(get_fits(file.get('basename'))) output = self.create_jpg_output(fits_paths, percent=0.9, cur_percent=0.0, color=True) + + output = {'output_files': output} else: output = {'output_files': []} raise ValueError('RGB Stack operation requires exactly 3 input files') From e9f5c01e33aeda636c59dd5a5e40b6c8f462c4bd Mon Sep 17 00:00:00 2001 From: Lloyd Dakin Date: Tue, 30 Jul 2024 11:46:21 -0700 Subject: [PATCH 3/5] removed random seed for testing --- datalab/datalab_session/data_operations/rgb_stack.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/datalab/datalab_session/data_operations/rgb_stack.py b/datalab/datalab_session/data_operations/rgb_stack.py index 4bea235..2894dad 100644 --- a/datalab/datalab_session/data_operations/rgb_stack.py +++ b/datalab/datalab_session/data_operations/rgb_stack.py @@ -50,14 +50,6 @@ def wizard_description(): 'minimum': 1, 'maximum': 1, 'filter': ['B'] - }, - 'random_seed': { - 'name': 'Seed', - 'description': 'so I can test this without it being cached', - 'type': 'number', - 'minimum': 0, - 'maximum': 99999.0, - 'default': 60.0 } } } From e74c63481619c25f0564b74e3777d050ff6330b0 Mon Sep 17 00:00:00 2001 From: Lloyd Dakin Date: Tue, 30 Jul 2024 16:18:43 -0700 Subject: [PATCH 4/5] fixed function descriptions, max dimensions for img, percent completion --- .../data_operations/data_operation.py | 14 +++++++------- .../datalab_session/data_operations/rgb_stack.py | 1 + datalab/datalab_session/util.py | 10 ++++------ 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/datalab/datalab_session/data_operations/data_operation.py b/datalab/datalab_session/data_operations/data_operation.py index b6a22e8..af8df83 100644 --- a/datalab/datalab_session/data_operations/data_operation.py +++ b/datalab/datalab_session/data_operations/data_operation.py @@ -9,7 +9,7 @@ import numpy as np from datalab.datalab_session.tasks import execute_data_operation -from datalab.datalab_session.util import add_file_to_bucket, get_hdu, fits_dimensions, stack_arrays, create_fits +from datalab.datalab_session.util import add_file_to_bucket, get_hdu, get_fits_dimensions, stack_arrays, create_fits CACHE_DURATION = 60 * 60 * 24 * 30 # cache for 30 days @@ -99,30 +99,30 @@ def set_failed(self, message: str): self.set_status('FAILED') self.set_message(message) - def create_jpg_output(self, fits_paths, percent=None, cur_percent=None, color=False, index=None) -> list: + def create_jpg_output(self, fits_paths: str, percent=None, cur_percent=None, color=False, index=None) -> list: """ Create jpgs from fits files and save them to S3 If using the color option fits_paths need to be in order R, G, B percent and cur_percent are used to update the progress of the operation """ - if type(fits_paths) is not list: + if not isinstance(fits_paths, list): fits_paths = [fits_paths] # create the jpgs from the fits files large_jpg_path = tempfile.NamedTemporaryFile(suffix=f'{self.cache_key}-large.jpg').name thumbnail_jpg_path = tempfile.NamedTemporaryFile(suffix=f'{self.cache_key}-small.jpg').name - height, width = fits_dimensions(fits_paths[0]) + max_height, max_width = max(get_fits_dimensions(path) for path in fits_paths) - fits_to_jpg(fits_paths, large_jpg_path, width=width, height=height, color=color) + fits_to_jpg(fits_paths, large_jpg_path, width=max_width, height=max_height, color=color) fits_to_jpg(fits_paths, thumbnail_jpg_path, color=color) # color photos take three files, so we store it as one fits file with a 3d SCI ndarray if color: arrays = [fits.open(file)['SCI'].data for file in fits_paths] - stacked = stack_arrays(arrays) - fits_file = create_fits(self.cache_key, stacked) + stacked_data = stack_arrays(arrays) + fits_file = create_fits(self.cache_key, stacked_data) else: fits_file = fits_paths[0] diff --git a/datalab/datalab_session/data_operations/rgb_stack.py b/datalab/datalab_session/data_operations/rgb_stack.py index 2894dad..fa76094 100644 --- a/datalab/datalab_session/data_operations/rgb_stack.py +++ b/datalab/datalab_session/data_operations/rgb_stack.py @@ -63,6 +63,7 @@ def operate(self): fits_paths = [] for file in rgb_input_list: fits_paths.append(get_fits(file.get('basename'))) + self.set_percent_completion(self.get_percent_completion() + 0.2) output = self.create_jpg_output(fits_paths, percent=0.9, cur_percent=0.0, color=True) diff --git a/datalab/datalab_session/util.py b/datalab/datalab_session/util.py index 0add14a..8d43a73 100644 --- a/datalab/datalab_session/util.py +++ b/datalab/datalab_session/util.py @@ -156,15 +156,13 @@ def get_hdu(basename: str, extension: str = 'SCI', source: str = 'archive') -> l return extension -def fits_dimensions(fits_file) -> tuple: - hdu = fits.open(fits_file) - height, width = hdu[1].shape - return height, width +def get_fits_dimensions(fits_file, extension: str = 'SCI') -> tuple: + return fits.open(fits_file)[extension].shape -def create_fits(key: str, image_arr: np.ndarray) -> fits.HDUList: +def create_fits(key: str, image_arr: np.ndarray) -> str: """ Creates a fits file with the given key and image array - Returns the HDUList and the dimensions of the image + Returns the the path to the fits_file """ header = fits.Header([('KEY', key)]) From 8ac1c7c9778ef9d421c369f33c2b53acd1001ada Mon Sep 17 00:00:00 2001 From: Lloyd Dakin Date: Thu, 1 Aug 2024 13:20:30 -0700 Subject: [PATCH 5/5] split create_jpg_outputs into two functions, create_jpg, and save output, moved functions from data operations to util file --- .../data_operations/data_operation.py | 56 +------------------ .../datalab_session/data_operations/median.py | 8 ++- .../data_operations/rgb_stack.py | 16 ++++-- datalab/datalab_session/util.py | 44 ++++++++++++++- 4 files changed, 60 insertions(+), 64 deletions(-) diff --git a/datalab/datalab_session/data_operations/data_operation.py b/datalab/datalab_session/data_operations/data_operation.py index af8df83..ae93f5d 100644 --- a/datalab/datalab_session/data_operations/data_operation.py +++ b/datalab/datalab_session/data_operations/data_operation.py @@ -1,15 +1,12 @@ from abc import ABC, abstractmethod import hashlib import json -import tempfile from django.core.cache import cache -from fits2image.conversions import fits_to_jpg -from astropy.io import fits import numpy as np from datalab.datalab_session.tasks import execute_data_operation -from datalab.datalab_session.util import add_file_to_bucket, get_hdu, get_fits_dimensions, stack_arrays, create_fits +from datalab.datalab_session.util import get_hdu CACHE_DURATION = 60 * 60 * 24 * 30 # cache for 30 days @@ -99,57 +96,6 @@ def set_failed(self, message: str): self.set_status('FAILED') self.set_message(message) - def create_jpg_output(self, fits_paths: str, percent=None, cur_percent=None, color=False, index=None) -> list: - """ - Create jpgs from fits files and save them to S3 - If using the color option fits_paths need to be in order R, G, B - percent and cur_percent are used to update the progress of the operation - """ - - if not isinstance(fits_paths, list): - fits_paths = [fits_paths] - - # create the jpgs from the fits files - large_jpg_path = tempfile.NamedTemporaryFile(suffix=f'{self.cache_key}-large.jpg').name - thumbnail_jpg_path = tempfile.NamedTemporaryFile(suffix=f'{self.cache_key}-small.jpg').name - - max_height, max_width = max(get_fits_dimensions(path) for path in fits_paths) - - fits_to_jpg(fits_paths, large_jpg_path, width=max_width, height=max_height, color=color) - fits_to_jpg(fits_paths, thumbnail_jpg_path, color=color) - - # color photos take three files, so we store it as one fits file with a 3d SCI ndarray - if color: - arrays = [fits.open(file)['SCI'].data for file in fits_paths] - stacked_data = stack_arrays(arrays) - fits_file = create_fits(self.cache_key, stacked_data) - else: - fits_file = fits_paths[0] - - - # Save Fits and Thumbnails in S3 Buckets - bucket_key = f'{self.cache_key}/{self.cache_key}-{index}' if index else f'{self.cache_key}/{self.cache_key}' - - fits_url = add_file_to_bucket(f'{bucket_key}.fits', fits_file) - large_jpg_url = add_file_to_bucket(f'{bucket_key}-large.jpg', large_jpg_path) - thumbnail_jpg_url = add_file_to_bucket(f'{bucket_key}-small.jpg', thumbnail_jpg_path) - - output = [] - output.append({ - 'fits_url': fits_url, - 'large_url': large_jpg_url, - 'thumbnail_url': thumbnail_jpg_url, - 'basename': f'{self.cache_key}', - 'source': 'datalab'} - ) - - if percent is not None and cur_percent is not None: - self.set_percent_completion(cur_percent + percent) - else: - self.set_percent_completion(0.9) - - return output - def get_fits_npdata(self, input_files: list[dict], percent=None, cur_percent=None) -> list[np.memmap]: total_files = len(input_files) image_data_list = [] diff --git a/datalab/datalab_session/data_operations/median.py b/datalab/datalab_session/data_operations/median.py index 55e94fd..732d948 100644 --- a/datalab/datalab_session/data_operations/median.py +++ b/datalab/datalab_session/data_operations/median.py @@ -3,7 +3,7 @@ import numpy as np from datalab.datalab_session.data_operations.data_operation import BaseDataOperation -from datalab.datalab_session.util import create_fits, stack_arrays +from datalab.datalab_session.util import create_fits, stack_arrays, create_jpgs, save_fits_and_thumbnails log = logging.getLogger() log.setLevel(logging.INFO) @@ -54,9 +54,11 @@ def operate(self): fits_file = create_fits(self.cache_key, median) - output = self.create_jpg_output(fits_file, percent=0.6, cur_percent=0.4) + large_jpg_path, small_jpg_path = create_jpgs(self.cache_key, fits_file) - output = {'output_files': output} + output_file = save_fits_and_thumbnails(self.cache_key, fits_file, large_jpg_path, small_jpg_path) + + output = {'output_files': [output_file]} else: output = {'output_files': []} diff --git a/datalab/datalab_session/data_operations/rgb_stack.py b/datalab/datalab_session/data_operations/rgb_stack.py index fa76094..6c8e8ce 100644 --- a/datalab/datalab_session/data_operations/rgb_stack.py +++ b/datalab/datalab_session/data_operations/rgb_stack.py @@ -1,10 +1,9 @@ import logging -import tempfile -from fits2image.conversions import fits_to_jpg +from astropy.io import fits from datalab.datalab_session.data_operations.data_operation import BaseDataOperation -from datalab.datalab_session.util import add_file_to_bucket, get_fits +from datalab.datalab_session.util import get_fits, stack_arrays, create_fits, save_fits_and_thumbnails, create_jpgs log = logging.getLogger() log.setLevel(logging.INFO) @@ -65,9 +64,16 @@ def operate(self): fits_paths.append(get_fits(file.get('basename'))) self.set_percent_completion(self.get_percent_completion() + 0.2) - output = self.create_jpg_output(fits_paths, percent=0.9, cur_percent=0.0, color=True) + large_jpg_path, small_jpg_path = create_jpgs(self.cache_key, fits_paths, color=True) - output = {'output_files': output} + # color photos take three files, so we store it as one fits file with a 3d SCI ndarray + arrays = [fits.open(file)['SCI'].data for file in fits_paths] + stacked_data = stack_arrays(arrays) + fits_file = create_fits(self.cache_key, stacked_data) + + output_file = save_fits_and_thumbnails(self.cache_key, fits_file, large_jpg_path, small_jpg_path) + + output = {'output_files': [output_file]} else: output = {'output_files': []} raise ValueError('RGB Stack operation requires exactly 3 input files') diff --git a/datalab/datalab_session/util.py b/datalab/datalab_session/util.py index 8d43a73..97c0431 100644 --- a/datalab/datalab_session/util.py +++ b/datalab/datalab_session/util.py @@ -7,9 +7,10 @@ import boto3 from astropy.io import fits import numpy as np +from botocore.exceptions import ClientError from django.conf import settings -from botocore.exceptions import ClientError +from fits2image.conversions import fits_to_jpg log = logging.getLogger() log.setLevel(logging.INFO) @@ -175,6 +176,47 @@ def create_fits(key: str, image_arr: np.ndarray) -> str: return fits_path +def create_jpgs(cache_key, fits_paths: str, color=False) -> list: + """ + Create jpgs from fits files and save them to S3 + If using the color option fits_paths need to be in order R, G, B + percent and cur_percent are used to update the progress of the operation + """ + + if not isinstance(fits_paths, list): + fits_paths = [fits_paths] + + # create the jpgs from the fits files + large_jpg_path = tempfile.NamedTemporaryFile(suffix=f'{cache_key}-large.jpg').name + thumbnail_jpg_path = tempfile.NamedTemporaryFile(suffix=f'{cache_key}-small.jpg').name + + max_height, max_width = max(get_fits_dimensions(path) for path in fits_paths) + + fits_to_jpg(fits_paths, large_jpg_path, width=max_width, height=max_height, color=color) + fits_to_jpg(fits_paths, thumbnail_jpg_path, color=color) + + return large_jpg_path, thumbnail_jpg_path + +def save_fits_and_thumbnails(cache_key, fits_path, large_jpg_path, thumbnail_jpg_path, index=None): + """ + Save Fits and Thumbnails in S3 Buckets, Returns the URLs in an output object + """ + bucket_key = f'{cache_key}/{cache_key}-{index}' if index else f'{cache_key}/{cache_key}' + + fits_url = add_file_to_bucket(f'{bucket_key}.fits', fits_path) + large_jpg_url = add_file_to_bucket(f'{bucket_key}-large.jpg', large_jpg_path) + thumbnail_jpg_url = add_file_to_bucket(f'{bucket_key}-small.jpg', thumbnail_jpg_path) + + output_file = dict({ + 'fits_url': fits_url, + 'large_url': large_jpg_url, + 'thumbnail_url': thumbnail_jpg_url, + 'basename': f'{cache_key}', + 'source': 'datalab'} + ) + + return output_file + def stack_arrays(array_list: list): """ Takes a list of numpy arrays, crops them to an equal shape, and stacks them to be a 3d numpy array