From f855d2047d68cde93371aa024a0220c9ea4eff17 Mon Sep 17 00:00:00 2001 From: Curtis McCully Date: Fri, 12 Apr 2024 11:27:32 -0400 Subject: [PATCH 1/3] Merge with upstream. --- CHANGES.md | 4 ++++ banzai/utils/fits_utils.py | 12 ++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 8393af55..00890822 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,7 @@ +1.15.2 (2023-04-12) +------------------- +- Fix to fpacking data when the image data array is None + 1.15.1 (2024-02-29) ------------------- - Minor fixes in photometry when there are bad pixels near the image edges diff --git a/banzai/utils/fits_utils.py b/banzai/utils/fits_utils.py index 9fba5a77..fe4ac70f 100755 --- a/banzai/utils/fits_utils.py +++ b/banzai/utils/fits_utils.py @@ -223,7 +223,11 @@ def pack(uncompressed_hdulist: fits.HDUList, lossless_extensions: Iterable) -> f quantize_level = 1e9 else: quantize_level = 64 - compressed_hdu = fits.CompImageHDU(data=np.ascontiguousarray(uncompressed_hdulist[0].data), + if uncompressed_hdulist[0].data is None: + data = None + else: + data = np.ascontiguousarray(uncompressed_hdulist[0].data) + compressed_hdu = fits.CompImageHDU(data=data, header=uncompressed_hdulist[0].header, quantize_level=quantize_level, quantize_method=1) hdulist = [primary_hdu, compressed_hdu] @@ -234,7 +238,11 @@ def pack(uncompressed_hdulist: fits.HDUList, lossless_extensions: Iterable) -> f quantize_level = 1e9 else: quantize_level = 64 - compressed_hdu = fits.CompImageHDU(data=np.ascontiguousarray(hdu.data), header=hdu.header, + if hdu.data is None: + data = None + else: + data = np.ascontiguousarray(hdu.data) + compressed_hdu = fits.CompImageHDU(data=data, header=hdu.header, quantize_level=quantize_level, quantize_method=1) hdulist.append(compressed_hdu) else: From 28afff4bb6cf447aa58deaaf46cc02643234f3d3 Mon Sep 17 00:00:00 2001 From: Curtis McCully Date: Fri, 10 Nov 2023 12:40:59 -0500 Subject: [PATCH 2/3] Fixes to opening fits files with no pixel data. --- banzai/utils/fits_utils.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/banzai/utils/fits_utils.py b/banzai/utils/fits_utils.py index fe4ac70f..9edb8b53 100755 --- a/banzai/utils/fits_utils.py +++ b/banzai/utils/fits_utils.py @@ -199,12 +199,15 @@ def unpack(compressed_hdulist: fits.HDUList) -> fits.HDUList: starting_extension = 1 for hdu in compressed_hdulist[starting_extension:]: if isinstance(hdu, fits.CompImageHDU): - data_type = str(hdu.data.dtype) - if 'int' == data_type[:3]: - data_type = getattr(np, 'u' + data_type) - data = np.array(hdu.data, data_type) + if hdu.data is None: + data = hdu.data else: - data = np.array(hdu.data, hdu.data.dtype) + data_type = str(hdu.data.dtype) + if 'int' == data_type[:3]: + data_type = getattr(np, 'u' + data_type) + data = np.array(hdu.data, data_type) + else: + data = np.array(hdu.data, hdu.data.dtype) hdulist.append(fits.ImageHDU(data=data, header=hdu.header)) elif isinstance(hdu, fits.BinTableHDU): hdulist.append(fits.BinTableHDU(data=hdu.data, header=hdu.header)) From de3c0b62b978b44aafb819b4209b37a48cb580cc Mon Sep 17 00:00:00 2001 From: Curtis McCully Date: Fri, 10 Nov 2023 13:15:51 -0500 Subject: [PATCH 3/3] Plumbed name through headeronly data types. --- banzai/data.py | 4 ++-- banzai/lco.py | 6 +++--- banzai/tests/test_munge.py | 6 +++--- banzai/tests/utils.py | 4 ++-- banzai/utils/realtime_utils.py | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/banzai/data.py b/banzai/data.py index 901a72a0..4adf7322 100644 --- a/banzai/data.py +++ b/banzai/data.py @@ -85,8 +85,8 @@ def to_fits(self, context) -> Union[fits.HDUList, list]: class HeaderOnly(Data): - def __init__(self, meta: Union[dict, fits.Header]): - super().__init__(data=np.zeros(0), meta=meta, memmap=False) + def __init__(self, meta: Union[dict, fits.Header], name): + super().__init__(data=np.zeros(0), meta=meta, memmap=False, name=name) def to_fits(self, context): return fits.HDUList([fits.ImageHDU(data=None, header=self.meta)]) diff --git a/banzai/lco.py b/banzai/lco.py index 8eb7fb78..8efd4d2b 100644 --- a/banzai/lco.py +++ b/banzai/lco.py @@ -412,7 +412,7 @@ def open(self, file_info, runtime_context) -> Optional[ObservationFrame]: for hdu in fits_hdu_list if hdu.data is not None): for hdu in fits_hdu_list: if hdu.data is None or hdu.data.size == 0: - hdu_list.append(HeaderOnly(meta=hdu.header)) + hdu_list.append(HeaderOnly(meta=hdu.header, name=hdu.header.get('EXTNAME'))) else: hdu_list.append(self.data_class(data=hdu.data, meta=hdu.header, name=hdu.header.get('EXTNAME'))) else: @@ -424,7 +424,7 @@ def open(self, file_info, runtime_context) -> Optional[ObservationFrame]: continue # Otherwise parse the fits file into a frame object and the corresponding data objects if hdu.data is None or hdu.data.size == 0: - hdu_list.append(HeaderOnly(meta=hdu.header)) + hdu_list.append(HeaderOnly(meta=hdu.header, name=hdu.header.get('EXTNAME'))) primary_hdu = hdu elif isinstance(hdu, fits.BinTableHDU): hdu_list.append(DataTable(data=Table(hdu.data), meta=hdu.header, name=hdu.header.get('EXTNAME'))) @@ -601,7 +601,7 @@ def _munge_data_cube(hdu): :return: List CCDData objects """ # The first extension gets to be a header only object - hdu_list = [HeaderOnly(meta=hdu.header)] + hdu_list = [HeaderOnly(meta=hdu.header, name=hdu.header.get('EXTNAME'))] # We need to properly set the datasec and detsec keywords in case we didn't read out the # middle row (the "Missing Row Problem"). diff --git a/banzai/tests/test_munge.py b/banzai/tests/test_munge.py index 6254e0a3..5cc2a7b2 100644 --- a/banzai/tests/test_munge.py +++ b/banzai/tests/test_munge.py @@ -23,7 +23,7 @@ def test_when_has_partial_coefficients(): for i in range(2): for j in range(2): header['CRSTLK{i}{j}'.format(i=i+1, j=j+1)] = 1.0 - hdu_list = [HeaderOnly(meta=header)] + [FakeCCDData() for i in range(4)] + hdu_list = [HeaderOnly(meta=header, name='')] + [FakeCCDData() for i in range(4)] with pytest.raises(MissingCrosstalkCoefficients): LCOFrameFactory._init_crosstalk(FakeLCOObservationFrame(hdu_list=hdu_list)) @@ -34,7 +34,7 @@ def test_when_has_coefficients(): for i in range(4): for j in range(4): header['CRSTLK{i}{j}'.format(i=i+1, j=j+1)] = 1.0 - hdu_list = [HeaderOnly(meta=header)] + [FakeCCDData() for i in range(4)] + hdu_list = [HeaderOnly(meta=header, name='')] + [FakeCCDData() for i in range(4)] LCOFrameFactory._init_crosstalk(FakeLCOObservationFrame(hdu_list=hdu_list)) @@ -44,7 +44,7 @@ def test_defaults_do_not_override_header(): for j in range(4): header['CRSTLK{i}{j}'.format(i=i+1, j=j+1)] = 1.0 - hdu_list = [HeaderOnly(meta=header)] + [FakeCCDData() for i in range(4)] + hdu_list = [HeaderOnly(meta=header, name='')] + [FakeCCDData() for i in range(4)] fake_image = FakeLCOObservationFrame(hdu_list=hdu_list) LCOFrameFactory._init_crosstalk(fake_image) diff --git a/banzai/tests/utils.py b/banzai/tests/utils.py index 486dc09a..a147ee97 100644 --- a/banzai/tests/utils.py +++ b/banzai/tests/utils.py @@ -102,8 +102,8 @@ def handles_inhomogeneous_set(stagetype, context, keyword, value, calibration_ma stage = stagetype(context) kwargs = {keyword: value} if calibration_maker: - images = [LCOCalibrationFrame(hdu_list=[HeaderOnly(meta=kwargs)])] - images += [LCOCalibrationFrame(hdu_list=[HeaderOnly()]) for x in range(6)] + images = [LCOCalibrationFrame(hdu_list=[HeaderOnly(meta=kwargs, name='')])] + images += [LCOCalibrationFrame(hdu_list=[HeaderOnly(meta={}, name=''),]) for x in range(6)] images = stage.do_stage(images) assert len(images) == 0 else: diff --git a/banzai/utils/realtime_utils.py b/banzai/utils/realtime_utils.py index 45684815..639f5146 100644 --- a/banzai/utils/realtime_utils.py +++ b/banzai/utils/realtime_utils.py @@ -86,7 +86,7 @@ def need_to_process_image(file_info, context): if 'frameid' in file_info: try: factory = import_utils.import_attribute(context.FRAME_FACTORY)() - test_image = factory.observation_frame_class(hdu_list=[HeaderOnly(file_info)], + test_image = factory.observation_frame_class(hdu_list=[HeaderOnly(file_info, name='')], file_path=file_info['filename']) test_image.instrument = factory.get_instrument_from_header(file_info, db_address=context.db_address) if image_utils.get_reduction_level(test_image.meta) != '00':