From 9ede7a390c34a6cfda8af8ab9291eb8685ba7e8f Mon Sep 17 00:00:00 2001 From: Robert Stein Date: Wed, 1 May 2024 19:59:28 -0700 Subject: [PATCH 01/25] midwayfalts midwayfalts Split up WINTER generator Alphabetise error stack More sorting Select dome flats Select dome flats working Nanfill midwayfalts midwayfalts Split up WINTER generator More sorting Select dome flats Select dome flats working Nanfill --- mirar/pipelines/winter/blocks.py | 19 ++++- mirar/pipelines/winter/generator/__init__.py | 1 + mirar/pipelines/winter/generator/reduce.py | 2 +- mirar/pipelines/winter/load_winter_image.py | 2 +- mirar/processors/flat.py | 82 ++++++++++++++++++++ mirar/processors/utils/__init__.py | 1 + mirar/processors/utils/nan_filler.py | 35 +++++++++ 7 files changed, 137 insertions(+), 5 deletions(-) create mode 100644 mirar/processors/utils/nan_filler.py diff --git a/mirar/pipelines/winter/blocks.py b/mirar/pipelines/winter/blocks.py index b588da47a..43106b19a 100644 --- a/mirar/pipelines/winter/blocks.py +++ b/mirar/pipelines/winter/blocks.py @@ -43,6 +43,8 @@ from mirar.pipelines.winter.generator import ( apply_rb_to_table, mask_stamps_around_bright_stars, + select_winter_dome_flats_images, + select_winter_flat_images, select_winter_sky_flat_images, winter_anet_sextractor_config_path_generator, winter_astrometric_ref_catalog_generator, @@ -149,6 +151,7 @@ ImageSaver, ImageSelector, MEFLoader, + NanFiller, ) from mirar.processors.utils.cal_hunter import CalHunter from mirar.processors.xmatch import XMatch @@ -315,7 +318,6 @@ load_unpacked = [ ImageLoader(input_sub_dir="raw_unpacked", input_img_dir=base_output_dir), - ImageRebatcher("EXPID"), CSVLog( export_keys=[ "UTCTIME", @@ -333,6 +335,7 @@ "T_ROIC", "FIELDID", "MEDCOUNT", + "DITHGRP", ] ), ImageRebatcher(BASE_NAME_KEY), @@ -359,7 +362,15 @@ ] flat_calibrate = [ + ImageRebatcher(["SUBCOORD", "FILTER"]), + FlatCalibrator( + cache_sub_dir="calibration_flats", + select_flat_images=select_winter_dome_flats_images, + cache_image_name_header_keys=["FILTER", "BOARD_ID"], + flat_mode="pixel", + ), ImageSelector((OBSCLASS_KEY, ["science"])), + ImageSaver(output_dir_name="domeflatcal"), ImageRebatcher( [ "BOARD_ID", @@ -374,9 +385,10 @@ FlatCalibrator( cache_sub_dir="sky_dither_flats", select_flat_images=select_winter_sky_flat_images, + flat_mode="structure", ), - ImageRebatcher(BASE_NAME_KEY), - ImageSaver(output_dir_name="skyflatcal"), + ImageSaver(output_dir_name="allskyflatcal"), + ImageRebatcher([BASE_NAME_KEY]), Sextractor( **sextractor_astrometry_config, write_regions_bool=True, @@ -469,6 +481,7 @@ stack_dithers = [ CustomImageBatchModifier(winter_boardid_6_demasker), ImageRebatcher("STACKID"), + NanFiller(), Swarp( swarp_config_path=swarp_config_path, calculate_dims_in_swarp=True, diff --git a/mirar/pipelines/winter/generator/__init__.py b/mirar/pipelines/winter/generator/__init__.py index 9946278ab..e3bde98bf 100644 --- a/mirar/pipelines/winter/generator/__init__.py +++ b/mirar/pipelines/winter/generator/__init__.py @@ -26,6 +26,7 @@ from mirar.pipelines.winter.generator.reduce import ( mask_stamps_around_bright_stars, select_winter_dome_flats_images, + select_winter_flat_images, select_winter_sky_flat_images, winter_anet_sextractor_config_path_generator, winter_boardid_6_demasker, diff --git a/mirar/pipelines/winter/generator/reduce.py b/mirar/pipelines/winter/generator/reduce.py index dbbe4aa05..321548005 100644 --- a/mirar/pipelines/winter/generator/reduce.py +++ b/mirar/pipelines/winter/generator/reduce.py @@ -18,7 +18,7 @@ OBSCLASS_KEY, REF_CAT_PATH_KEY, SATURATE_KEY, - get_output_dir, + get_output_dir ) from mirar.pipelines.winter.config import sextractor_anet_config from mirar.pipelines.winter.fourier_bkg_model import subtract_fourier_background_model diff --git a/mirar/pipelines/winter/load_winter_image.py b/mirar/pipelines/winter/load_winter_image.py index d76b97c59..b1be7837f 100644 --- a/mirar/pipelines/winter/load_winter_image.py +++ b/mirar/pipelines/winter/load_winter_image.py @@ -579,7 +579,7 @@ def get_raw_winter_mask(image: Image) -> np.ndarray: mask[1060:, :] = 1.0 # Mask a vertical strip - mask[:, 998:1002] = 1.0 + mask[:, 992:1002] = 1.0 # Mask another vertical strip mask[:, 1266:1273] = 1.0 diff --git a/mirar/processors/flat.py b/mirar/processors/flat.py index 66e2b6ba1..74fc32a17 100644 --- a/mirar/processors/flat.py +++ b/mirar/processors/flat.py @@ -9,6 +9,15 @@ from copy import copy import numpy as np +<<<<<<< HEAD +<<<<<<< HEAD +from astropy.convolution import Tophat2DKernel, convolve_fft +======= +from astropy.convolution import Tophat2DKernel, convolve, convolve_fft +>>>>>>> 60685c68 (working) +======= +from astropy.convolution import Tophat2DKernel, convolve_fft +>>>>>>> 822f21d8 (Nanfill) from mirar.data import Image, ImageBatch from mirar.errors import ImageNotFoundError @@ -26,6 +35,28 @@ logger = logging.getLogger(__name__) +def get_convolution(data: np.ndarray, kernel_width: int) -> np.ndarray: + """ + Convolve data with a tophat kernel + + :param data: Image data + :param kernel_width: Width of the kernel (pixels) + :return: Smoothed image + """ + pad_top = np.array([data[0] for _ in range(kernel_width)]) + pad_bottom = np.array([data[-1] for _ in range(kernel_width)]) + extended = np.vstack([pad_top, data, pad_bottom]) + pad_left = np.array([extended.T[0] for _ in range(kernel_width)]) + pad_right = np.array([extended.T[-1] for _ in range(kernel_width)]) + extended = np.hstack([pad_left.T, extended, pad_right.T]) + + tophat_kernel = Tophat2DKernel(kernel_width) + smooth_illumination = convolve_fft( + extended, tophat_kernel, nan_treatment="interpolate" + )[kernel_width:-kernel_width, kernel_width:-kernel_width] + return smooth_illumination + + class MissingFlatError(ImageNotFoundError): """ Error for when a dark image is missing @@ -61,6 +92,7 @@ def __init__( flat_nan_threshold: float = 0.0, select_flat_images: Callable[[ImageBatch], ImageBatch] = default_select_flat, flat_mask_key: str = None, + flat_mode: str = "median", **kwargs, ): super().__init__(*args, **kwargs) @@ -71,6 +103,9 @@ def __init__( self.flat_nan_threshold = flat_nan_threshold self.select_cache_images = select_flat_images self.flat_mask_key = flat_mask_key + self.flat_mode = flat_mode + if not self.flat_mode in ["median", "pixel", "structure"]: + raise ValueError(f"Flat mode {self.flat_mode} not supported") def description(self) -> str: return "Creates a flat image, divides other images by this image." @@ -152,6 +187,53 @@ def make_image( master_flat = np.nanmedian(flats, axis=2) + if self.flat_mode != "median": + + if self.flat_mode == "pixel": + flatdata_norm_smooth = get_convolution(master_flat, 40) + + pixel_variation = master_flat / flatdata_norm_smooth + + # Clip outliers (they'll get worked out in stacking) + std = np.nanstd(pixel_variation) + sig = abs(pixel_variation - np.nanmedian(pixel_variation)) / std + + mask = sig > 2.0 + + logger.debug( + f"Masking {np.sum(mask)} pixels " + f"out of {len(mask.flatten()) }in flat" + ) + pixel_variation[mask] = np.nan + master_flat = pixel_variation / np.nanmedian(pixel_variation) + + elif self.flat_mode == "structure": + flatdata_norm_smooth = get_convolution(master_flat, 100) + flatdata_norm_smooth[np.isnan(master_flat)] = np.nan + + pixel_variation = master_flat / flatdata_norm_smooth + + # Clip outliers (they'll get worked out in stacking) + std = np.nanstd(pixel_variation) + sig = abs(pixel_variation - np.nanmedian(pixel_variation)) / std + + mask = sig > 3.0 + + logger.debug( + f"Masking {np.sum(mask)} pixels " + f"out of {len(mask.flatten()) }in flat" + ) + + pixel_variation[sig > 3.0] = np.nan + master_flat = ( + pixel_variation + * flatdata_norm_smooth + / np.nanmedian(pixel_variation) + ) + + else: + raise ValueError(f"Flat mode {self.flat_mode} not supported") + master_flat_image = Image(master_flat, header=copy(images[0].get_header())) master_flat_image[COADD_KEY] = n_frames diff --git a/mirar/processors/utils/__init__.py b/mirar/processors/utils/__init__.py index e1b578d5f..195dc4b26 100644 --- a/mirar/processors/utils/__init__.py +++ b/mirar/processors/utils/__init__.py @@ -17,3 +17,4 @@ select_from_images, ) from mirar.processors.utils.multi_ext_parser import MultiExtParser +from mirar.processors.utils.nan_filler import NanFiller diff --git a/mirar/processors/utils/nan_filler.py b/mirar/processors/utils/nan_filler.py new file mode 100644 index 000000000..20ab4fd54 --- /dev/null +++ b/mirar/processors/utils/nan_filler.py @@ -0,0 +1,35 @@ +""" +Module for filling NaN values in an image +""" + +import logging + +import numpy as np + +from mirar.data import ImageBatch +from mirar.processors.base_processor import BaseImageProcessor + +logger = logging.getLogger(__name__) + + +class NanFiller(BaseImageProcessor): + """ + Processor to fill image nans + """ + + base_key = "nanfill" + + def __str__(self): + return "Processor to pad masked image regions with the median." + + def _apply_to_images( + self, + batch: ImageBatch, + ) -> ImageBatch: + + for image in batch: + data = image.get_data() + data[np.isnan(data)] = np.nanmedian(data) + image.set_data(data) + + return batch From a7b180ce1005ade05810236d94eaf04d2d7f669b Mon Sep 17 00:00:00 2001 From: Robert Stein Date: Fri, 10 May 2024 16:58:50 -0700 Subject: [PATCH 02/25] Progress --- mirar/processors/flat.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/mirar/processors/flat.py b/mirar/processors/flat.py index 74fc32a17..c11829f52 100644 --- a/mirar/processors/flat.py +++ b/mirar/processors/flat.py @@ -9,15 +9,7 @@ from copy import copy import numpy as np -<<<<<<< HEAD -<<<<<<< HEAD from astropy.convolution import Tophat2DKernel, convolve_fft -======= -from astropy.convolution import Tophat2DKernel, convolve, convolve_fft ->>>>>>> 60685c68 (working) -======= -from astropy.convolution import Tophat2DKernel, convolve_fft ->>>>>>> 822f21d8 (Nanfill) from mirar.data import Image, ImageBatch from mirar.errors import ImageNotFoundError @@ -104,7 +96,7 @@ def __init__( self.select_cache_images = select_flat_images self.flat_mask_key = flat_mask_key self.flat_mode = flat_mode - if not self.flat_mode in ["median", "pixel", "structure"]: + if self.flat_mode not in ["median", "pixel", "structure"]: raise ValueError(f"Flat mode {self.flat_mode} not supported") def description(self) -> str: From c8f54c59fb34a56a419a2a224a7d04f9d5cf2923 Mon Sep 17 00:00:00 2001 From: Robert Stein Date: Sat, 11 May 2024 08:25:47 -0700 Subject: [PATCH 03/25] Normal working version --- mirar/pipelines/winter/blocks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mirar/pipelines/winter/blocks.py b/mirar/pipelines/winter/blocks.py index 43106b19a..93bf4fb59 100644 --- a/mirar/pipelines/winter/blocks.py +++ b/mirar/pipelines/winter/blocks.py @@ -318,6 +318,7 @@ load_unpacked = [ ImageLoader(input_sub_dir="raw_unpacked", input_img_dir=base_output_dir), + ImageRebatcher("UTCTIME"), CSVLog( export_keys=[ "UTCTIME", @@ -386,6 +387,7 @@ cache_sub_dir="sky_dither_flats", select_flat_images=select_winter_sky_flat_images, flat_mode="structure", + # flat_mode="median", ), ImageSaver(output_dir_name="allskyflatcal"), ImageRebatcher([BASE_NAME_KEY]), From c7c6058632855cad100439b6d55b49ebf77f1a6b Mon Sep 17 00:00:00 2001 From: Robert Stein Date: Tue, 14 May 2024 11:03:23 -0700 Subject: [PATCH 04/25] interim --- mirar/pipelines/winter/blocks.py | 3 +-- mirar/pipelines/winter/config/__init__.py | 3 +++ mirar/pipelines/winter/load_winter_image.py | 7 +++++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/mirar/pipelines/winter/blocks.py b/mirar/pipelines/winter/blocks.py index 93bf4fb59..5ed55a6e1 100644 --- a/mirar/pipelines/winter/blocks.py +++ b/mirar/pipelines/winter/blocks.py @@ -336,7 +336,6 @@ "T_ROIC", "FIELDID", "MEDCOUNT", - "DITHGRP", ] ), ImageRebatcher(BASE_NAME_KEY), @@ -387,7 +386,7 @@ cache_sub_dir="sky_dither_flats", select_flat_images=select_winter_sky_flat_images, flat_mode="structure", - # flat_mode="median", + # flat_mode="median", ), ImageSaver(output_dir_name="allskyflatcal"), ImageRebatcher([BASE_NAME_KEY]), diff --git a/mirar/pipelines/winter/config/__init__.py b/mirar/pipelines/winter/config/__init__.py index 0f2760ab6..4f8e1df5b 100644 --- a/mirar/pipelines/winter/config/__init__.py +++ b/mirar/pipelines/winter/config/__init__.py @@ -88,6 +88,9 @@ "5.0", # Hs flats ], ), + CalRequirement( + target_name="domeflat", required_field="FILTER", required_values=["Y", "J", "Hs"] + ), ] winter_avro_schema_path = winter_file_dir.joinpath("avro_schema/winter.alert.avsc") diff --git a/mirar/pipelines/winter/load_winter_image.py b/mirar/pipelines/winter/load_winter_image.py index b1be7837f..f2cb26615 100644 --- a/mirar/pipelines/winter/load_winter_image.py +++ b/mirar/pipelines/winter/load_winter_image.py @@ -128,6 +128,13 @@ def clean_header(header: fits.Header) -> fits.Header: else: header["MIRCOVER"] = None + # Throw out flats with low counts + + if header[OBSCLASS_KEY].lower() == "flat": + if header["MEDCOUNT"] < 20000.0: + header[OBSCLASS_KEY] = "test" + header[TARGET_KEY] = "test" + header["EXPTIME"] = np.rint(header["EXPTIME"]) # Set up the target name From 87ac1bed42dc96a429548c938e6fb92756b591de Mon Sep 17 00:00:00 2001 From: Robert Stein Date: Mon, 17 Jun 2024 16:33:51 -0700 Subject: [PATCH 05/25] Fix typo in cal requirements --- mirar/pipelines/winter/config/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirar/pipelines/winter/config/__init__.py b/mirar/pipelines/winter/config/__init__.py index 4f8e1df5b..5cd042690 100644 --- a/mirar/pipelines/winter/config/__init__.py +++ b/mirar/pipelines/winter/config/__init__.py @@ -89,7 +89,7 @@ ], ), CalRequirement( - target_name="domeflat", required_field="FILTER", required_values=["Y", "J", "Hs"] + target_name="flat", required_field="FILTER", required_values=["Y", "J", "H"] ), ] From 7d2013415f4f7407de868813544d0841ba47bfa8 Mon Sep 17 00:00:00 2001 From: Robert Stein Date: Tue, 25 Jun 2024 12:40:08 -0700 Subject: [PATCH 06/25] interim nlc --- mirar/pipelines/winter/blocks.py | 29 ++++++++++++- mirar/pipelines/winter/load_winter_image.py | 7 ---- mirar/pipelines/winter/nlc.py | 20 +++++++++ mirar/pipelines/winter/winter_pipeline.py | 2 + mirar/processors/flat.py | 6 +-- mirar/processors/utils/__init__.py | 1 + mirar/processors/utils/mode_masker.py | 46 +++++++++++++++++++++ mirar/processors/utils/nan_filler.py | 2 +- 8 files changed, 100 insertions(+), 13 deletions(-) create mode 100644 mirar/pipelines/winter/nlc.py create mode 100644 mirar/processors/utils/mode_masker.py diff --git a/mirar/pipelines/winter/blocks.py b/mirar/pipelines/winter/blocks.py index 5ed55a6e1..3bef72795 100644 --- a/mirar/pipelines/winter/blocks.py +++ b/mirar/pipelines/winter/blocks.py @@ -93,6 +93,7 @@ Source, Stack, ) +from mirar.pipelines.winter.nlc import apply_winter_nlc from mirar.pipelines.winter.validator import ( masked_images_rejector, poor_astrometric_quality_rejector, @@ -151,6 +152,7 @@ ImageSaver, ImageSelector, MEFLoader, + ModeMasker, NanFiller, ) from mirar.processors.utils.cal_hunter import CalHunter @@ -225,7 +227,9 @@ input_sub_dir="raw", load_image=load_winter_mef_image, ), - CalHunter(load_image=load_winter_mef_image, requirements=winter_cal_requirements), + CalHunter( + load_image=load_winter_mef_image, requirements=winter_cal_requirements + ), # FIXME: add back in ] load_astrometry = [ @@ -339,6 +343,12 @@ ] ), ImageRebatcher(BASE_NAME_KEY), + # FIXME: Remove this line + ImageSelector( + ("FILTER", ["dark", "J"]), + ("PROGNAME", ["2024A000", "2023A014"]), + # ("TARGET", ["dark", "flat", "request_2024A000_2024_06_12_01_16_59.db_0"]) + ), ] export_unpacked = [DatabaseImageInserter(db_table=Raw, duplicate_protocol="replace")] @@ -347,6 +357,11 @@ # Detrend blocks +non_linear_correction = [ + ImageRebatcher(BASE_NAME_KEY), + CustomImageBatchModifier(apply_winter_nlc), +] + dark_calibrate = [ ImageRebatcher( ["BOARD_ID", EXPTIME_KEY, "SUBCOORD", "GAINCOLT", "GAINCOLB", "GAINROW"] @@ -483,6 +498,8 @@ CustomImageBatchModifier(winter_boardid_6_demasker), ImageRebatcher("STACKID"), NanFiller(), + MaskPixelsFromFunction(mask_function=get_raw_winter_mask), + ImageSaver(output_dir_name="prestack"), Swarp( swarp_config_path=swarp_config_path, calculate_dims_in_swarp=True, @@ -495,9 +512,16 @@ min_required_coadds=3, ), ImageRebatcher(BASE_NAME_KEY), + ModeMasker(), ImageSaver(output_dir_name="stack"), ] +remask = [ + ImageLoader(input_sub_dir="stack", input_img_dir=base_output_dir), + ModeMasker(), # Mask out the pixels which are stacked nans + ImageSaver(output_dir_name="stack_masks"), +] + photcal_and_export = [ HeaderAnnotator(input_keys=LATEST_SAVE_KEY, output_key=RAW_IMG_KEY), CustomImageBatchModifier(masked_images_rejector), @@ -906,7 +930,8 @@ unpack_all = load_raw + extract_all + csvlog + mask_and_split + save_raw full_reduction = ( - dark_calibrate + non_linear_correction + + dark_calibrate + flat_calibrate + fourier_filter + process_and_stack diff --git a/mirar/pipelines/winter/load_winter_image.py b/mirar/pipelines/winter/load_winter_image.py index f2cb26615..b1be7837f 100644 --- a/mirar/pipelines/winter/load_winter_image.py +++ b/mirar/pipelines/winter/load_winter_image.py @@ -128,13 +128,6 @@ def clean_header(header: fits.Header) -> fits.Header: else: header["MIRCOVER"] = None - # Throw out flats with low counts - - if header[OBSCLASS_KEY].lower() == "flat": - if header["MEDCOUNT"] < 20000.0: - header[OBSCLASS_KEY] = "test" - header[TARGET_KEY] = "test" - header["EXPTIME"] = np.rint(header["EXPTIME"]) # Set up the target name diff --git a/mirar/pipelines/winter/nlc.py b/mirar/pipelines/winter/nlc.py new file mode 100644 index 000000000..5989d0dfa --- /dev/null +++ b/mirar/pipelines/winter/nlc.py @@ -0,0 +1,20 @@ +from winternlc.non_linear_correction import nlc_single + +from mirar.data import ImageBatch + + +def apply_winter_nlc(images: ImageBatch) -> ImageBatch: + """ + Apply WINTER non-linear correction to images + + :param images: ImageBatch to apply non-linear correction to + :return: Corrected ImageBatch + """ + + for image in images: + data = image.get_data() + board_id = image["BOARD_ID"] + corrected_image = nlc_single(data, board_id) + image.set_data(corrected_image) + + return images diff --git a/mirar/pipelines/winter/winter_pipeline.py b/mirar/pipelines/winter/winter_pipeline.py index 1c288c62f..4ee339a90 100644 --- a/mirar/pipelines/winter/winter_pipeline.py +++ b/mirar/pipelines/winter/winter_pipeline.py @@ -40,6 +40,7 @@ reduce_unpacked_subset, refbuild, reftest, + remask, save_raw, select_split_subset, send_to_skyportal, @@ -102,6 +103,7 @@ class WINTERPipeline(Pipeline): + detect_candidates + process_candidates + avro_broadcast, + "remask": remask, "default_subset": reduce_unpacked_subset + imsub + detect_candidates diff --git a/mirar/processors/flat.py b/mirar/processors/flat.py index c11829f52..63a055448 100644 --- a/mirar/processors/flat.py +++ b/mirar/processors/flat.py @@ -190,7 +190,7 @@ def make_image( std = np.nanstd(pixel_variation) sig = abs(pixel_variation - np.nanmedian(pixel_variation)) / std - mask = sig > 2.0 + mask = sig > 1.0 logger.debug( f"Masking {np.sum(mask)} pixels " @@ -209,14 +209,14 @@ def make_image( std = np.nanstd(pixel_variation) sig = abs(pixel_variation - np.nanmedian(pixel_variation)) / std - mask = sig > 3.0 + mask = sig > 1.0 logger.debug( f"Masking {np.sum(mask)} pixels " f"out of {len(mask.flatten()) }in flat" ) - pixel_variation[sig > 3.0] = np.nan + pixel_variation[mask] = np.nan master_flat = ( pixel_variation * flatdata_norm_smooth diff --git a/mirar/processors/utils/__init__.py b/mirar/processors/utils/__init__.py index 195dc4b26..6d56be028 100644 --- a/mirar/processors/utils/__init__.py +++ b/mirar/processors/utils/__init__.py @@ -16,5 +16,6 @@ ImageSelector, select_from_images, ) +from mirar.processors.utils.mode_masker import ModeMasker from mirar.processors.utils.multi_ext_parser import MultiExtParser from mirar.processors.utils.nan_filler import NanFiller diff --git a/mirar/processors/utils/mode_masker.py b/mirar/processors/utils/mode_masker.py new file mode 100644 index 000000000..73dbd1f3d --- /dev/null +++ b/mirar/processors/utils/mode_masker.py @@ -0,0 +1,46 @@ +""" +Module for filling NaN values into an image +""" + +import logging + +import numpy as np +from scipy.stats import mode + +from mirar.data import ImageBatch +from mirar.processors.base_processor import BaseImageProcessor + +logger = logging.getLogger(__name__) + + +class ModeMasker(BaseImageProcessor): + """ + Processor to mask the most common value in an image + """ + + base_key = "modemask" + + def description(self): + return "Processor to mask image pixels with very common values" + + def _apply_to_images( + self, + batch: ImageBatch, + ) -> ImageBatch: + + for image in batch: + data = image.get_data() + + n_pixels = data.size + + mode_value = mode(data[~np.isnan(data)])[0] + frac = np.sum(data == mode_value) / n_pixels + + while frac > 0.0001: + data[data == mode_value] = np.nan + mode_value = mode(data[~np.isnan(data)])[0] + frac = np.sum(data == mode_value) / n_pixels + + image.set_data(data) + + return batch diff --git a/mirar/processors/utils/nan_filler.py b/mirar/processors/utils/nan_filler.py index 20ab4fd54..b377faaa6 100644 --- a/mirar/processors/utils/nan_filler.py +++ b/mirar/processors/utils/nan_filler.py @@ -19,7 +19,7 @@ class NanFiller(BaseImageProcessor): base_key = "nanfill" - def __str__(self): + def description(self): return "Processor to pad masked image regions with the median." def _apply_to_images( From a57a8a71bcf434b5b76330d3003d2c6c716ccd6d Mon Sep 17 00:00:00 2001 From: Robert Stein Date: Thu, 1 Aug 2024 08:54:45 -0600 Subject: [PATCH 07/25] Mid --- mirar/pipelines/winter/blocks.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/mirar/pipelines/winter/blocks.py b/mirar/pipelines/winter/blocks.py index 3bef72795..095f0aead 100644 --- a/mirar/pipelines/winter/blocks.py +++ b/mirar/pipelines/winter/blocks.py @@ -343,12 +343,6 @@ ] ), ImageRebatcher(BASE_NAME_KEY), - # FIXME: Remove this line - ImageSelector( - ("FILTER", ["dark", "J"]), - ("PROGNAME", ["2024A000", "2023A014"]), - # ("TARGET", ["dark", "flat", "request_2024A000_2024_06_12_01_16_59.db_0"]) - ), ] export_unpacked = [DatabaseImageInserter(db_table=Raw, duplicate_protocol="replace")] From cd60fdd26072c9b2021d8053877f76b6f0200b9b Mon Sep 17 00:00:00 2001 From: Robert Stein Date: Fri, 15 Nov 2024 13:32:19 -0500 Subject: [PATCH 08/25] Fix download --- poetry.lock | 22 +++++++++++++++++++++- pyproject.toml | 1 + 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 8dfd1f786..80319a146 100644 --- a/poetry.lock +++ b/poetry.lock @@ -5792,6 +5792,26 @@ files = [ {file = "widgetsnbextension-4.0.13.tar.gz", hash = "sha256:ffcb67bc9febd10234a362795f643927f4e0c05d9342c727b65d2384f8feacb6"}, ] +[[package]] +name = "winternlc" +version = "1.1.0" +description = "" +optional = false +python-versions = ">=3.10" +files = [ + {file = "winternlc-1.1.0-py3-none-any.whl", hash = "sha256:f6acd7f0e551b8ef8ec57129a2ea38e6c7de625267e7f1d8f03d754f5df23c8b"}, +] + +[package.dependencies] +astropy = "*" +matplotlib = "*" +numpy = "*" +pre-commit = "*" +scipy = "*" + +[package.extras] +dev = ["black (==24.4.2)", "coveralls", "isort (==5.13.2)", "pylint (==3.2.2)"] + [[package]] name = "winterrb" version = "1.0.0" @@ -5871,4 +5891,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.10.0,<3.13" -content-hash = "8d5811a0980504f155dcbd932233c3ab7cda76d115f81723365e07391aff1b8b" +content-hash = "b99f31f166df0bb24bb3f3caaeda4c9b272f2c805ab29f4a883fbfc2e3a4d09c" diff --git a/pyproject.toml b/pyproject.toml index 9f4f915ac..59944ccb2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ python-dotenv = "^1.0.1" pyarrow = ">=15.0.2,<19.0.0" torch = "^2.2.0,<2.3.0" winterrb = "^1.0.0" +winternlc = "^1.1.0" [tool.poetry.group.docs.dependencies] sphinx = ">=7.0.1,<9.0.0" From 5d08085d509a67d79c14b6c7f2558c3e873caa31 Mon Sep 17 00:00:00 2001 From: Robert Stein Date: Fri, 15 Nov 2024 13:35:07 -0500 Subject: [PATCH 09/25] Isortify --- mirar/pipelines/winter/generator/reduce.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirar/pipelines/winter/generator/reduce.py b/mirar/pipelines/winter/generator/reduce.py index 321548005..dbbe4aa05 100644 --- a/mirar/pipelines/winter/generator/reduce.py +++ b/mirar/pipelines/winter/generator/reduce.py @@ -18,7 +18,7 @@ OBSCLASS_KEY, REF_CAT_PATH_KEY, SATURATE_KEY, - get_output_dir + get_output_dir, ) from mirar.pipelines.winter.config import sextractor_anet_config from mirar.pipelines.winter.fourier_bkg_model import subtract_fourier_background_model From 37bc2fa9a87522f7f0322a4b0e840238872da18a Mon Sep 17 00:00:00 2001 From: Robert Stein Date: Wed, 27 Nov 2024 14:36:48 -0500 Subject: [PATCH 10/25] New nlc mode --- mirar/pipelines/winter/nlc.py | 15 ++++++++++----- poetry.lock | 7 ++++--- pyproject.toml | 2 +- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/mirar/pipelines/winter/nlc.py b/mirar/pipelines/winter/nlc.py index 5989d0dfa..2ffbc6277 100644 --- a/mirar/pipelines/winter/nlc.py +++ b/mirar/pipelines/winter/nlc.py @@ -1,11 +1,16 @@ -from winternlc.non_linear_correction import nlc_single +""" +Module for applying WINTER non-linear correction to images +""" + +from winternlc.non_linear_correction import apply_nlc_single from mirar.data import ImageBatch def apply_winter_nlc(images: ImageBatch) -> ImageBatch: """ - Apply WINTER non-linear correction to images + Apply WINTER non-linear correction to images. + Uses header information to apply the correct correction. :param images: ImageBatch to apply non-linear correction to :return: Corrected ImageBatch @@ -13,8 +18,8 @@ def apply_winter_nlc(images: ImageBatch) -> ImageBatch: for image in images: data = image.get_data() - board_id = image["BOARD_ID"] - corrected_image = nlc_single(data, board_id) - image.set_data(corrected_image) + header = image.get_header() + corrected_data = apply_nlc_single(data, header) + image.set_data(corrected_data) return images diff --git a/poetry.lock b/poetry.lock index 80319a146..a9bf7c0fc 100644 --- a/poetry.lock +++ b/poetry.lock @@ -5794,12 +5794,13 @@ files = [ [[package]] name = "winternlc" -version = "1.1.0" +version = "1.3.0" description = "" optional = false python-versions = ">=3.10" files = [ - {file = "winternlc-1.1.0-py3-none-any.whl", hash = "sha256:f6acd7f0e551b8ef8ec57129a2ea38e6c7de625267e7f1d8f03d754f5df23c8b"}, + {file = "winternlc-1.3.0-py3-none-any.whl", hash = "sha256:dcd9cb36ef6cec184ff172e49ceeeef21fbdd03bf1723809470e226a984b8d67"}, + {file = "winternlc-1.3.0.tar.gz", hash = "sha256:ccc76851eb4a7d0c32103e2ddec99daac81477d333f59a0778c681e3f77b3ba5"}, ] [package.dependencies] @@ -5891,4 +5892,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.10.0,<3.13" -content-hash = "b99f31f166df0bb24bb3f3caaeda4c9b272f2c805ab29f4a883fbfc2e3a4d09c" +content-hash = "bd0c0fba9d7f7c62fa5e84d5fe842337538423c36bcde9d361fae68c1d0742ca" diff --git a/pyproject.toml b/pyproject.toml index 59944ccb2..6f1dc2826 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,7 @@ python-dotenv = "^1.0.1" pyarrow = ">=15.0.2,<19.0.0" torch = "^2.2.0,<2.3.0" winterrb = "^1.0.0" -winternlc = "^1.1.0" +winternlc = "^1.3.0" [tool.poetry.group.docs.dependencies] sphinx = ">=7.0.1,<9.0.0" From 4a28adb742d80fd7a0cc31617187a858b510e304 Mon Sep 17 00:00:00 2001 From: Robert Stein Date: Wed, 27 Nov 2024 15:06:02 -0500 Subject: [PATCH 11/25] Only round if t>1 --- mirar/pipelines/winter/load_winter_image.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mirar/pipelines/winter/load_winter_image.py b/mirar/pipelines/winter/load_winter_image.py index b1be7837f..0c401fae7 100644 --- a/mirar/pipelines/winter/load_winter_image.py +++ b/mirar/pipelines/winter/load_winter_image.py @@ -128,7 +128,10 @@ def clean_header(header: fits.Header) -> fits.Header: else: header["MIRCOVER"] = None - header["EXPTIME"] = np.rint(header["EXPTIME"]) + if float(header["EXPTIME"]) >= 1.0: + header["EXPTIME"] = np.rint(header["EXPTIME"]) + else: + header["EXPTIME"] = np.round(header["EXPTIME"], 2) # Set up the target name From 05f7a2deb67ff5ca89ee3caba8a655ee1f2553ef Mon Sep 17 00:00:00 2001 From: Robert Stein Date: Wed, 27 Nov 2024 16:40:22 -0500 Subject: [PATCH 12/25] Fix typo --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6f1dc2826..095a6a6b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -112,7 +112,7 @@ extension-pkg-whitelist=["pydantic"] generated-members=["u.*"] [tool.poetry.scripts] -mirar-run = 'mirar.__main__:main' +mirar-run = 'mirar.__main__' mirar-docs-autogen = "mirar.utils.docs:iterate_rst_generation" winter-stack = 'mirar.pipelines.winter.run:run_stack_of_stacks' From 99ddc8f9cbb2fc2476d3ada038e6ea00cb5167a5 Mon Sep 17 00:00:00 2001 From: Robert Stein Date: Wed, 27 Nov 2024 16:46:23 -0500 Subject: [PATCH 13/25] Update poetry.lock --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index a9bf7c0fc..3bcd5e0bc 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1458,13 +1458,13 @@ zstandard = ["zstandard"] [[package]] name = "fastjsonschema" -version = "2.20.0" +version = "2.21.0" description = "Fastest Python implementation of JSON schema" optional = false python-versions = "*" files = [ - {file = "fastjsonschema-2.20.0-py3-none-any.whl", hash = "sha256:5875f0b0fa7a0043a91e93a9b8f793bcbbba9691e7fd83dca95c28ba26d21f0a"}, - {file = "fastjsonschema-2.20.0.tar.gz", hash = "sha256:3d48fc5300ee96f5d116f10fe6f28d938e6008f59a6a025c2649475b87f76a23"}, + {file = "fastjsonschema-2.21.0-py3-none-any.whl", hash = "sha256:5b23b8e7c9c6adc0ecb91c03a0768cb48cd154d9159378a69c8318532e0b5cbf"}, + {file = "fastjsonschema-2.21.0.tar.gz", hash = "sha256:a02026bbbedc83729da3bfff215564b71902757f33f60089f1abae193daa4771"}, ] [package.extras] From 641fcd9dc9de2c65695f128079728b253c06b39c Mon Sep 17 00:00:00 2001 From: Robert Stein Date: Wed, 27 Nov 2024 16:51:40 -0500 Subject: [PATCH 14/25] fix script --- mirar/__main__.py | 316 ++++++++++++++++++++++++---------------------- pyproject.toml | 2 +- 2 files changed, 167 insertions(+), 151 deletions(-) diff --git a/mirar/__main__.py b/mirar/__main__.py index 913b700ca..ed3a84d30 100644 --- a/mirar/__main__.py +++ b/mirar/__main__.py @@ -22,173 +22,189 @@ logger = logging.getLogger(__name__) -parser = argparse.ArgumentParser( - description=f"{PACKAGE_NAME}: Modular Image Reduction and Analysis Resource" -) - -parser.add_argument( - "-n", "--night", default=None, help="Sub-directory to use in the data directory" -) -parser.add_argument("-p", "--pipeline", help="Pipeline to be used", required=True) -parser.add_argument( - "-c", "--config", default=None, help="Pipeline configuration to be used" -) -parser.add_argument( - "-pc", "--postprocessconfig", default=None, help="Pipeline configuration to be used" -) -parser.add_argument( - "--logfile", - default=None, - help="If a path is passed, all logs will be written to this file.", -) -parser.add_argument("--level", default="INFO", help="Python logging level") -parser.add_argument( - "-b", "--batch", default=None, help="Only process a specific image batch" -) -parser.add_argument( - "--download", help="Download images from server", action="store_true", default=False -) -parser.add_argument( - "--failfast", help="Fail on first error", action="store_true", default=False -) - -parser.add_argument("-m", "--monitor", action="store_true", default=False) -parser.add_argument( - "--emailrecipients", - default=None, - help="Spaceless comma-separated values of email recipients", -) -parser.add_argument( - "--emailsender", - default=None, - help="One email sender", -) -parser.add_argument( - "--midwaypostprocesshours", - default=16.0, - help="Time, in hours, to wait before sending a summary email", -) -parser.add_argument( - "--finalpostprocesshours", - default=48.0, - help="Time, in hours, to wait before ceasing monitoring for new images", -) -parser.add_argument( - "--rawdir", - default=RAW_IMG_SUB_DIR, - help="Subdirectory to look in for raw images of a given night", -) - -args = parser.parse_args() - -if args.download: - Pipeline.pipelines[args.pipeline.lower()].download_raw_images_for_night( - night=args.night - ) - logger.info("Download complete") +def main(): + """ + Main executable for mirar. You can execute the code from the terminal like: + + .. codeblock:: bash + python -m mirar -args... + """ + parser = argparse.ArgumentParser( + description=f"{PACKAGE_NAME}: Modular Image Reduction and Analysis Resource" + ) -night = args.night + parser.add_argument( + "-n", "--night", default=None, help="Sub-directory to use in the data directory" + ) + parser.add_argument("-p", "--pipeline", help="Pipeline to be used", required=True) + parser.add_argument( + "-c", "--config", default=None, help="Pipeline configuration to be used" + ) + parser.add_argument( + "-pc", + "--postprocessconfig", + default=None, + help="Pipeline configuration to be used", + ) + parser.add_argument( + "--logfile", + default=None, + help="If a path is passed, all logs will be written to this file.", + ) + parser.add_argument("--level", default="INFO", help="Python logging level") + parser.add_argument( + "-b", "--batch", default=None, help="Only process a specific image batch" + ) + parser.add_argument( + "--download", + help="Download images from server", + action="store_true", + default=False, + ) + parser.add_argument( + "--failfast", help="Fail on first error", action="store_true", default=False + ) -with tempfile.TemporaryDirectory(dir=TEMP_DIR) as temp_dir_path: - print(f"Using cache {temp_dir_path}") + parser.add_argument("-m", "--monitor", action="store_true", default=False) + parser.add_argument( + "--emailrecipients", + default=None, + help="Spaceless comma-separated values of email recipients", + ) + parser.add_argument( + "--emailsender", + default=None, + help="One email sender", + ) + parser.add_argument( + "--midwaypostprocesshours", + default=16.0, + help="Time, in hours, to wait before sending a summary email", + ) + parser.add_argument( + "--finalpostprocesshours", + default=48.0, + help="Time, in hours, to wait before ceasing monitoring for new images", + ) + parser.add_argument( + "--rawdir", + default=RAW_IMG_SUB_DIR, + help="Subdirectory to look in for raw images of a given night", + ) - cache.set_cache_dir(temp_dir_path) + args = parser.parse_args() - if args.monitor: - if args.emailrecipients is not None: - EMAIL_RECIPIENTS = args.emailrecipients.split(",") - else: - EMAIL_RECIPIENTS = None - - CONFIG = args.config - if CONFIG is None: - CONFIG = "realtime" - - if night is None: - ln = Time.now() - night = str(ln).split(" ", maxsplit=1)[0].replace("-", "") - - monitor = Monitor( - pipeline=args.pipeline, - night=night, - realtime_configurations=CONFIG, - postprocess_configurations=( - args.postprocessconfig.split(",") - if args.postprocessconfig is not None - else None - ), - log_level=args.level, - final_postprocess_hours=args.finalpostprocesshours, - midway_postprocess_hours=args.midwaypostprocesshours, - email_sender=args.emailsender, - email_recipients=EMAIL_RECIPIENTS, - raw_dir=args.rawdir, + if args.download: + Pipeline.pipelines[args.pipeline.lower()].download_raw_images_for_night( + night=args.night ) - monitor.process_realtime() - else: - # Set up logging - - log = logging.getLogger("mirar") + logger.info("Download complete") + + night = args.night + + with tempfile.TemporaryDirectory(dir=TEMP_DIR) as temp_dir_path: + print(f"Using cache {temp_dir_path}") + + cache.set_cache_dir(temp_dir_path) + + if args.monitor: + if args.emailrecipients is not None: + email_recipients = args.emailrecipients.split(",") + else: + email_recipients = None + + config = args.config + if config is None: + config = "realtime" + + if night is None: + ln = Time.now() + night = str(ln).split(" ", maxsplit=1)[0].replace("-", "") + + monitor = Monitor( + pipeline=args.pipeline, + night=night, + realtime_configurations=config, + postprocess_configurations=( + args.postprocessconfig.split(",") + if args.postprocessconfig is not None + else None + ), + log_level=args.level, + final_postprocess_hours=args.finalpostprocesshours, + midway_postprocess_hours=args.midwaypostprocesshours, + email_sender=args.emailsender, + email_recipients=email_recipients, + raw_dir=args.rawdir, + ) + monitor.process_realtime() - if args.logfile is None: - handler = logging.StreamHandler(sys.stdout) else: - handler = logging.FileHandler(args.logfile) + # Set up logging - formatter = logging.Formatter( - "%(name)s [l %(lineno)d] - %(levelname)s - %(message)s" - ) - handler.setFormatter(formatter) - log.addHandler(handler) - log.setLevel(args.level) - - CONFIG = args.config - if CONFIG is None: - CONFIG = "default" - - if night is None: - ln = Time.now() - 1.0 * u.day - night = str(ln).split(" ", maxsplit=1)[0].replace("-", "") - - pipe = get_pipeline( - args.pipeline, - selected_configurations=CONFIG, - night=night, - ) + log = logging.getLogger("mirar") - batches, errorstack = pipe.reduce_images( - catch_all_errors=not args.failfast, - ) + if args.logfile is None: + handler = logging.StreamHandler(sys.stdout) + else: + handler = logging.FileHandler(args.logfile) - processors = pipe.get_latest_configuration() - flowify(processors, pipe.get_flowchart_output_path(), include_stats=True) + formatter = logging.Formatter( + "%(name)s [l %(lineno)d] - %(levelname)s - %(message)s" + ) + handler.setFormatter(formatter) + log.addHandler(handler) + log.setLevel(args.level) + + config = args.config + if config is None: + config = "default" + + if night is None: + ln = Time.now() - 1.0 * u.day + night = str(ln).split(" ", maxsplit=1)[0].replace("-", "") + + pipe = get_pipeline( + args.pipeline, + selected_configurations=config, + night=night, + ) - if args.postprocessconfig is not None: - post_config = [ - x for x in pipe.set_configuration(CONFIG) if isinstance(x, ImageLoader) - ][:1] - post_config += pipe.postprocess_configuration( - errorstack=errorstack, - selected_configurations=args.postprocessconfig.split(","), + _, errorstack = pipe.reduce_images( + catch_all_errors=not args.failfast, ) - PROTECTED_KEY = "_new_postprocess" + processors = pipe.get_latest_configuration() + flowify(processors, pipe.get_flowchart_output_path(), include_stats=True) - pipe.add_configuration(PROTECTED_KEY, post_config) - pipe.set_configuration(PROTECTED_KEY) + if args.postprocessconfig is not None: + post_config = [ + x + for x in pipe.set_configuration(config) + if isinstance(x, ImageLoader) + ][:1] + post_config += pipe.postprocess_configuration( + errorstack=errorstack, + selected_configurations=args.postprocessconfig.split(","), + ) - _, new_errorstack, _ = pipe.reduce_images( - selected_configurations=PROTECTED_KEY, - catch_all_errors=True, - ) - errorstack += new_errorstack + protected_key = "_new_postprocess" - print(errorstack.summarise_error_stack(verbose=False)) + pipe.add_configuration(protected_key, post_config) + pipe.set_configuration(protected_key) - errorstack.summarise_error_stack( - output_path=pipe.get_error_output_path(), verbose=True - ) + _, new_errorstack, _ = pipe.reduce_images( + selected_configurations=protected_key, + catch_all_errors=True, + ) + errorstack += new_errorstack + + print(errorstack.summarise_error_stack(verbose=False)) + + errorstack.summarise_error_stack( + output_path=pipe.get_error_output_path(), verbose=True + ) - logger.info("End of mirar execution") + logger.info("End of mirar execution") diff --git a/pyproject.toml b/pyproject.toml index 095a6a6b0..6f1dc2826 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -112,7 +112,7 @@ extension-pkg-whitelist=["pydantic"] generated-members=["u.*"] [tool.poetry.scripts] -mirar-run = 'mirar.__main__' +mirar-run = 'mirar.__main__:main' mirar-docs-autogen = "mirar.utils.docs:iterate_rst_generation" winter-stack = 'mirar.pipelines.winter.run:run_stack_of_stacks' From c54e57d6cf462a9d4fba0e2a4954627d548e3803 Mon Sep 17 00:00:00 2001 From: Robert Stein Date: Mon, 2 Dec 2024 12:56:23 -0500 Subject: [PATCH 15/25] Update to skyflat config --- mirar/pipelines/winter/blocks.py | 35 +++++++++++++++++++++-- mirar/pipelines/winter/config/__init__.py | 17 ++++++++--- mirar/pipelines/winter/winter_pipeline.py | 6 ++++ 3 files changed, 51 insertions(+), 7 deletions(-) diff --git a/mirar/pipelines/winter/blocks.py b/mirar/pipelines/winter/blocks.py index 095f0aead..cfe2123e3 100644 --- a/mirar/pipelines/winter/blocks.py +++ b/mirar/pipelines/winter/blocks.py @@ -25,6 +25,7 @@ base_output_dir, ) from mirar.pipelines.winter.config import ( + base_winter_cal_requirements, prv_candidate_cols, psfex_path, scamp_config_path, @@ -223,13 +224,21 @@ # Loading load_raw = [ + MEFLoader( + input_sub_dir="raw", + load_image=load_winter_mef_image, + ), + CalHunter(load_image=load_winter_mef_image, requirements=winter_cal_requirements), +] + +load_raw_no_dome_flats = [ MEFLoader( input_sub_dir="raw", load_image=load_winter_mef_image, ), CalHunter( - load_image=load_winter_mef_image, requirements=winter_cal_requirements - ), # FIXME: add back in + load_image=load_winter_mef_image, requirements=base_winter_cal_requirements + ), ] load_astrometry = [ @@ -370,7 +379,7 @@ CustomImageBatchModifier(winter_dark_oversubtraction_rejector), ] -flat_calibrate = [ +dome_flats = [ ImageRebatcher(["SUBCOORD", "FILTER"]), FlatCalibrator( cache_sub_dir="calibration_flats", @@ -380,6 +389,9 @@ ), ImageSelector((OBSCLASS_KEY, ["science"])), ImageSaver(output_dir_name="domeflatcal"), +] + +sky_flats = [ ImageRebatcher( [ "BOARD_ID", @@ -409,6 +421,8 @@ ImageSaver(output_dir_name="skysub"), ] +flat_calibrate = dome_flats + sky_flats + load_calibrated = [ ImageLoader(input_sub_dir="skysub", input_img_dir=base_output_dir), ImageBatcher(["UTCTIME", "BOARD_ID"]), @@ -923,6 +937,10 @@ unpack_all = load_raw + extract_all + csvlog + mask_and_split + save_raw +unpack_all_no_dome_flats = ( + load_raw_no_dome_flats + extract_all + csvlog + mask_and_split + save_raw +) + full_reduction = ( non_linear_correction + dark_calibrate @@ -932,6 +950,15 @@ + photcal_and_export ) +full_reduction_no_dome_flats = ( + non_linear_correction + + dark_calibrate + + sky_flats # Only sky flats + + fourier_filter + + process_and_stack + + photcal_and_export +) + photcal_stacks = [ ImageLoader( input_sub_dir="stack", @@ -948,6 +975,8 @@ reduce = unpack_all + full_reduction +reduce_no_dome_flats = unpack_all_no_dome_flats + full_reduction_no_dome_flats + reftest = ( unpack_subset + dark_calibrate diff --git a/mirar/pipelines/winter/config/__init__.py b/mirar/pipelines/winter/config/__init__.py index 5cd042690..41df836f4 100644 --- a/mirar/pipelines/winter/config/__init__.py +++ b/mirar/pipelines/winter/config/__init__.py @@ -76,16 +76,25 @@ psfex_path = winter_file_dir.joinpath("photom.psfex") ref_psfex_path = winter_file_dir.joinpath("reference.psfex") -winter_cal_requirements = [ +base_winter_cal_requirements = [ CalRequirement( target_name="dark", required_field="EXPTIME", required_values=[ "120.0", # J/Y "60.0", # Hs - "3.0", # J flats - "4.0", # Y flats - "5.0", # Hs flats + ], + ), +] + +winter_cal_requirements = base_winter_cal_requirements + [ + CalRequirement( + target_name="dark", + required_field="EXPTIME", + required_values=[ + "0.35", # J flats + "0.57", # Y flats + "0.46", # Hs flats ], ), CalRequirement( diff --git a/mirar/pipelines/winter/winter_pipeline.py b/mirar/pipelines/winter/winter_pipeline.py index 4ee339a90..90dd42e52 100644 --- a/mirar/pipelines/winter/winter_pipeline.py +++ b/mirar/pipelines/winter/winter_pipeline.py @@ -36,6 +36,7 @@ process_candidates, realtime, reduce, + reduce_no_dome_flats, reduce_unpacked, reduce_unpacked_subset, refbuild, @@ -103,6 +104,11 @@ class WINTERPipeline(Pipeline): + detect_candidates + process_candidates + avro_broadcast, + "skyflat": reduce_no_dome_flats + + imsub + + detect_candidates + + process_candidates + + avro_broadcast, "remask": remask, "default_subset": reduce_unpacked_subset + imsub From 4740e1d61d46a2d6269c08c31cf64d2b7be3c663 Mon Sep 17 00:00:00 2001 From: Robert Stein Date: Mon, 2 Dec 2024 12:58:57 -0500 Subject: [PATCH 16/25] move imageselector --- mirar/pipelines/winter/blocks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirar/pipelines/winter/blocks.py b/mirar/pipelines/winter/blocks.py index cfe2123e3..1b09faae5 100644 --- a/mirar/pipelines/winter/blocks.py +++ b/mirar/pipelines/winter/blocks.py @@ -387,11 +387,11 @@ cache_image_name_header_keys=["FILTER", "BOARD_ID"], flat_mode="pixel", ), - ImageSelector((OBSCLASS_KEY, ["science"])), ImageSaver(output_dir_name="domeflatcal"), ] sky_flats = [ + ImageSelector((OBSCLASS_KEY, ["science"])), ImageRebatcher( [ "BOARD_ID", From d80b79c5f85ddc7626cfbaf53f87f4aec0c3b916 Mon Sep 17 00:00:00 2001 From: Robert Stein Date: Mon, 2 Dec 2024 13:32:40 -0500 Subject: [PATCH 17/25] Fallback old flat mode --- mirar/pipelines/winter/blocks.py | 39 ++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/mirar/pipelines/winter/blocks.py b/mirar/pipelines/winter/blocks.py index 1b09faae5..4e0f0cbae 100644 --- a/mirar/pipelines/winter/blocks.py +++ b/mirar/pipelines/winter/blocks.py @@ -379,7 +379,7 @@ CustomImageBatchModifier(winter_dark_oversubtraction_rejector), ] -dome_flats = [ +flat_calibrate = [ ImageRebatcher(["SUBCOORD", "FILTER"]), FlatCalibrator( cache_sub_dir="calibration_flats", @@ -387,11 +387,8 @@ cache_image_name_header_keys=["FILTER", "BOARD_ID"], flat_mode="pixel", ), - ImageSaver(output_dir_name="domeflatcal"), -] - -sky_flats = [ ImageSelector((OBSCLASS_KEY, ["science"])), + ImageSaver(output_dir_name="domeflatcal"), ImageRebatcher( [ "BOARD_ID", @@ -421,7 +418,35 @@ ImageSaver(output_dir_name="skysub"), ] -flat_calibrate = dome_flats + sky_flats +sky_flat_calibrate = [ + ImageSelector((OBSCLASS_KEY, ["science"])), + ImageRebatcher( + [ + "BOARD_ID", + "FILTER", + "SUBCOORD", + "GAINCOLT", + "GAINCOLB", + "GAINROW", + TARGET_KEY, + ] + ), + FlatCalibrator( + cache_sub_dir="sky_dither_flats", + select_flat_images=select_winter_sky_flat_images, + flat_mode="median", + ), + ImageRebatcher(BASE_NAME_KEY), + ImageSaver(output_dir_name="skyflatcal"), + Sextractor( + **sextractor_astrometry_config, + write_regions_bool=True, + output_sub_dir="skysub", + checkimage_type=["-BACKGROUND"], + ), + SextractorBkgSubtractor(), + ImageSaver(output_dir_name="skysub"), +] load_calibrated = [ ImageLoader(input_sub_dir="skysub", input_img_dir=base_output_dir), @@ -953,7 +978,7 @@ full_reduction_no_dome_flats = ( non_linear_correction + dark_calibrate - + sky_flats # Only sky flats + + sky_flat_calibrate # Only sky flats + fourier_filter + process_and_stack + photcal_and_export From 6e05859259dc4f7736dbd91d3a6fd34fd156bbdb Mon Sep 17 00:00:00 2001 From: Robert Stein Date: Wed, 4 Dec 2024 15:30:43 -0500 Subject: [PATCH 18/25] Update --- mirar/pipelines/winter/blocks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mirar/pipelines/winter/blocks.py b/mirar/pipelines/winter/blocks.py index 4e0f0cbae..0c088677c 100644 --- a/mirar/pipelines/winter/blocks.py +++ b/mirar/pipelines/winter/blocks.py @@ -403,8 +403,8 @@ FlatCalibrator( cache_sub_dir="sky_dither_flats", select_flat_images=select_winter_sky_flat_images, - flat_mode="structure", - # flat_mode="median", + # flat_mode="structure", + flat_mode="median", ), ImageSaver(output_dir_name="allskyflatcal"), ImageRebatcher([BASE_NAME_KEY]), From bcb95e896e8801ce738ca38a1e5ffeb1b70d1ce6 Mon Sep 17 00:00:00 2001 From: Robert Stein Date: Wed, 4 Dec 2024 15:31:02 -0500 Subject: [PATCH 19/25] Fix db user set up --- mirar/database/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirar/database/setup.py b/mirar/database/setup.py index 6a0212e7f..b2d08b055 100644 --- a/mirar/database/setup.py +++ b/mirar/database/setup.py @@ -35,7 +35,7 @@ def setup_database(db_base: Union[DeclarativeBase, BaseTable]): f"Failed to validate credentials for user {DB_USER}. " f"Will try creating new user with this name using admin credentials." ) - pg_admin = PostgresAdmin() + pg_admin = PostgresAdmin(db_name="postgres") pg_admin.validate_credentials() pg_admin.create_new_user(new_db_user=DB_USER, new_password=DB_PASSWORD) pg_user.validate_credentials() From 676d5055ecc03961a0b5eaa2a863006c4ba3ed08 Mon Sep 17 00:00:00 2001 From: Robert Stein Date: Wed, 4 Dec 2024 15:34:34 -0500 Subject: [PATCH 20/25] Fix db user set up --- mirar/database/setup.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/mirar/database/setup.py b/mirar/database/setup.py index b2d08b055..d34f814b3 100644 --- a/mirar/database/setup.py +++ b/mirar/database/setup.py @@ -31,14 +31,18 @@ def setup_database(db_base: Union[DeclarativeBase, BaseTable]): try: pg_user.validate_credentials() except OperationalError: - logger.warning( - f"Failed to validate credentials for user {DB_USER}. " - f"Will try creating new user with this name using admin credentials." - ) - pg_admin = PostgresAdmin(db_name="postgres") - pg_admin.validate_credentials() - pg_admin.create_new_user(new_db_user=DB_USER, new_password=DB_PASSWORD) - pg_user.validate_credentials() + alt_pg_user = PostgresUser(db_name="postgres") + try: + alt_pg_user.validate_credentials() + except OperationalError: + logger.warning( + f"Failed to validate credentials for user {DB_USER}. " + f"Will try creating new user with this name using admin credentials." + ) + pg_admin = PostgresAdmin(db_name="postgres") + pg_admin.validate_credentials() + pg_admin.create_new_user(new_db_user=DB_USER, new_password=DB_PASSWORD) + pg_user.validate_credentials() pg_user.create_db(db_name=db_name) From c60cb62c6fb164121324832fb249e898ac46c201 Mon Sep 17 00:00:00 2001 From: Robert Stein Date: Wed, 4 Dec 2024 15:35:25 -0500 Subject: [PATCH 21/25] Fix db user set up --- mirar/database/setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mirar/database/setup.py b/mirar/database/setup.py index d34f814b3..380ee58a0 100644 --- a/mirar/database/setup.py +++ b/mirar/database/setup.py @@ -29,12 +29,15 @@ def setup_database(db_base: Union[DeclarativeBase, BaseTable]): pg_user = PostgresUser() try: + # Check if user + db exists pg_user.validate_credentials() except OperationalError: alt_pg_user = PostgresUser(db_name="postgres") try: + # Check if user exists in 'postgres' db instead alt_pg_user.validate_credentials() except OperationalError: + # Create new user logger.warning( f"Failed to validate credentials for user {DB_USER}. " f"Will try creating new user with this name using admin credentials." From b6afcd8540ebefdf27aa1faf7fa5f5ce8e63c650 Mon Sep 17 00:00:00 2001 From: Robert Stein Date: Thu, 5 Dec 2024 10:54:26 -0500 Subject: [PATCH 22/25] More careful checking --- mirar/pipelines/winter/load_winter_image.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mirar/pipelines/winter/load_winter_image.py b/mirar/pipelines/winter/load_winter_image.py index 0c401fae7..8a72ea537 100644 --- a/mirar/pipelines/winter/load_winter_image.py +++ b/mirar/pipelines/winter/load_winter_image.py @@ -241,17 +241,17 @@ def clean_header(header: fits.Header) -> fits.Header: if "NUMDITHS" not in header.keys(): header["NUMDITHS"] = None - else: + elif header["NUMDITHS"] is not None: header["NUMDITHS"] = int(header["NUMDITHS"]) if "DITHNUM" not in header.keys(): header["DITHNUM"] = None - else: + elif header["DITHNUM"] is not None: header["DITHNUM"] = int(header["DITHNUM"]) if "DITHSTEP" not in header.keys(): header["DITHSTEP"] = None - else: + elif header["DITHSTEP"] is not None: header["DITHSTEP"] = float(header["DITHSTEP"]) try: From 48d04ba85f26585b5e1161f8fc1c126288141b9b Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Thu, 16 Jan 2025 13:08:17 -0500 Subject: [PATCH 23/25] save nlcified images --- mirar/pipelines/winter/blocks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mirar/pipelines/winter/blocks.py b/mirar/pipelines/winter/blocks.py index 0c088677c..f41b8483f 100644 --- a/mirar/pipelines/winter/blocks.py +++ b/mirar/pipelines/winter/blocks.py @@ -363,6 +363,7 @@ non_linear_correction = [ ImageRebatcher(BASE_NAME_KEY), CustomImageBatchModifier(apply_winter_nlc), + ImageSaver(output_dir_name="nlc_corrected"), ] dark_calibrate = [ From 820782912d41379f587292ba654dd1502df2cdd9 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Fri, 17 Jan 2025 20:41:23 -0800 Subject: [PATCH 24/25] add main call --- mirar/__main__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mirar/__main__.py b/mirar/__main__.py index ed3a84d30..fbd2468f8 100644 --- a/mirar/__main__.py +++ b/mirar/__main__.py @@ -208,3 +208,6 @@ def main(): ) logger.info("End of mirar execution") + + +main() From 83836c40797d477b7935d56091cbdef96077b545 Mon Sep 17 00:00:00 2001 From: Robert Stein Date: Wed, 22 Jan 2025 13:30:56 -0500 Subject: [PATCH 25/25] Fix main --- mirar/__main__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mirar/__main__.py b/mirar/__main__.py index fbd2468f8..3ae262159 100644 --- a/mirar/__main__.py +++ b/mirar/__main__.py @@ -210,4 +210,5 @@ def main(): logger.info("End of mirar execution") -main() +if __name__ == "__main__": + main()