Merge pull request #16 from LCOGT/fix/line-analysis-median-lco-archiv…

…e-integration

Line Analysis Bug, Median Memmap Bug, Local Development Auth token changes

datalab/datalab_session/analysis/line_profile.py

@@ -1,14 +1,20 @@
 from skimage.measure import profile_line
 
-from datalab.datalab_session.util import scale_points
+from datalab.datalab_session.util import scale_flip_points
 
 # For creating an array of brightness along a user drawn line
 def line_profile(input: dict, sci_hdu: object):
   """
     Creates an array of luminosity values and the length of the line in arcseconds
   """
-  points = scale_points(input['width'], input['height'], sci_hdu.data, [(input["x1"], input["y1"]), (input["x2"], input["y2"])])
+  points = scale_flip_points(input['width'], input['height'], sci_hdu.data, [(input["x1"], input["y1"]), (input["x2"], input["y2"])])
   line_profile = profile_line(sci_hdu.data, points[0], points[1], mode="constant", cval=-1)
   arcsec = len(line_profile) * sci_hdu.header["PIXSCALE"]
 
   return {"line_profile": line_profile, "arcsec": arcsec}
+
+def debug_point_on_sci_data(x, y, sci_hdu: object):
+  """
+    Debugging function to check a point (x,y) on the sci data has the same value as the point cross checked in DS9
+  """
+  print(f"data: {sci_hdu.data[y, x]}")

datalab/datalab_session/data_operations/data_operation.py

@@ -136,30 +136,23 @@ def create_and_store_fits(self, hdu_list: fits.HDUList, percent=None, cur_percen
 
     def get_fits_npdata(self, input_files: list[dict], percent=None, cur_percent=None) -> list[np.memmap]:
         total_files = len(input_files)
-        memmap_paths = []
-
-        # get the fits urls, download their file, extract the image data, and store in a list
-        with tempfile.TemporaryDirectory() as temp_dir:
-            for index, file_info in enumerate(input_files, start=1):
-                basename = file_info.get('basename', 'No basename found')
-                archive_record = get_archive_from_basename(basename)
-
-                try:
-                    fits_url = archive_record[0].get('url', 'No URL found')
-                except IndexError:
-                    continue
-
-                with fits.open(fits_url) as hdu_list:
-                    data = hdu_list['SCI'].data
-                    memmap_path = os.path.join(temp_dir, f'memmap_{index}.dat')
-                    memmap_array = np.memmap(memmap_path, dtype=data.dtype, mode='w+', shape=data.shape)
-                    memmap_array[:] = data[:]
-                    memmap_paths.append(memmap_path)
-
-                if percent is not None and cur_percent is not None:
-                    self.set_percent_completion(cur_percent + index/total_files * percent)
-
-            return [
-                np.memmap(path, dtype=np.float32, mode='r', shape=memmap_array.shape)
-                for path in memmap_paths
-            ]
+        image_data_list = []
+
+        # get the fits urls and extract the image data
+        for index, file_info in enumerate(input_files, start=1):
+            basename = file_info.get('basename', 'No basename found')
+            archive_record = get_archive_from_basename(basename)
+
+            try:
+                fits_url = archive_record[0].get('url', 'No URL found')
+            except Exception as e:
+                raise FileNotFoundError(f"No image found with specified basename: {basename} Error: {e}")
+
+            with fits.open(fits_url) as hdu_list:
+                data = hdu_list['SCI'].data
+                image_data_list.append(data)
+
+            if percent is not None and cur_percent is not None:
+                self.set_percent_completion(cur_percent + index/total_files * percent)
+
+        return image_data_list

datalab/datalab_session/util.py

@@ -87,14 +87,17 @@ def get_archive_from_basename(basename: str) -> dict:
   """
   query_params = {'basename_exact': basename }
 
-  response = requests.get(settings.ARCHIVE_API + '/frames/', params=query_params)
+  headers = {
+    'Authorization': f'Token {settings.ARCHIVE_API_TOKEN}'
+  }
+
+  response = requests.get(settings.ARCHIVE_API + '/frames/', params=query_params, headers=headers)
 
   try:
     image_data = response.json()
     results = image_data.get('results', None)
-  except IndexError:
-    log.error(f"No image found with specified basename: {basename}")
-    raise FileNotFoundError
+  except Exception as e:
+    raise FileNotFoundError(f"Error fetching image data from archive: {e}")
 
   return results
 
@@ -111,11 +114,10 @@ def get_hdu(basename: str, extension: str = 'SCI') -> list[fits.HDUList]:
 
   try:
     fits_url = archive_record[0].get('url', 'No URL found')
-  except IndexError:
-    RuntimeWarning(f"No image found with specified basename: {basename}")
-
-  hdu = fits.open(fits_url)
-  return hdu[extension]
+    hdu = fits.open(fits_url)
+    return hdu[extension]
+  except Exception as e:
+    raise FileNotFoundError(f"No image found with specified basename: {basename} Error: {e}")
 
 def create_fits(key: str, image_arr: np.ndarray) -> fits.HDUList:
 
@@ -139,12 +141,11 @@ def stack_arrays(array_list: list):
 
   return stacked
 
-def scale_points(small_img_width: int, small_img_height: int, img_array: list, points: list[tuple[int, int]]):
+def scale_flip_points(small_img_width: int, small_img_height: int, img_array: list, points: list[tuple[int, int]]):
   """
     Scale the coordinates from a smaller image to the full sized fits so we know the positions of the coords on the 2dnumpy array
     Returns the list of tuple points with coords scaled for the numpy array
   """
-
   large_height, large_width = np.shape(img_array)
 
   # If the aspect ratios don't match we can't be certain where the point was
@@ -156,5 +157,7 @@ def scale_points(small_img_width: int, small_img_height: int, img_array: list, p
 
   points_array = np.array(points)
   scaled_points = np.int_(points_array * [width_scale, height_scale])
+  # html origin is top left, numpy origin is bottom left, so we need to flip the y axis
+  scaled_points[:, 1] = large_height - scaled_points[:, 1]
 
   return scaled_points

datalab/settings.py

@@ -134,7 +134,8 @@ def get_list_from_env(variable, default=None):
 DATALAB_OPERATION_BUCKET = os.getenv('DATALAB_OPERATION_BUCKET', 'datalab-operation-output-bucket')
 
 # Datalab Archive
-ARCHIVE_API = os.getenv('ARCHIVE_API', 'https://datalab-archive.photonranch.org')
+ARCHIVE_API = os.getenv('ARCHIVE_API', 'https://archive-api.lco.global')
+ARCHIVE_API_TOKEN = os.getenv('ARCHIVE_API_TOKEN')
 
 # Database
 # https://docs.djangoproject.com/en/4.2/ref/settings/#databases