From c0b745fcf929070139343655491ea5c17c9702d7 Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Mon, 2 Dec 2024 12:08:33 -0800 Subject: [PATCH 1/4] Added truth_to_observed pipeline --- .../degradation/truth_to_observed.py | 80 +++++++++++++++++++ tests/astro_tools/test_pipline.py | 1 + 2 files changed, 81 insertions(+) create mode 100644 src/rail/pipelines/degradation/truth_to_observed.py diff --git a/src/rail/pipelines/degradation/truth_to_observed.py b/src/rail/pipelines/degradation/truth_to_observed.py new file mode 100644 index 0000000..7bfd821 --- /dev/null +++ b/src/rail/pipelines/degradation/truth_to_observed.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python +# coding: utf-8 + +# Prerquisites, os, and numpy +import os +import numpy as np + +# Various rail modules +from rail.tools.photometry_tools import Dereddener, Reddener + +from rail.core.stage import RailStage, RailPipeline + +import ceci + +from rail.core.utils import RAILDIR + +if 'PZ_DUSTMAP_DIR' not in os.environ: + os.environ['PZ_DUSTMAP_DIR'] = '.' + +dustmap_dir = os.path.expandvars("${PZ_DUSTMAP_DIR}") + +from .spectroscopic_selection_pipeline.py import SELECTORS, CommonConfigParams +from .apply_phot_errors import ERROR_MODELS + + +class TruthToObservedPipeline(RailPipeline): + + default_input_dict = dict(input='dummy.in') + + def __init__(self, error_models=None, selectors=None, blending=False)): + RailPipeline.__init__(self) + + DS = RailStage.data_store + DS.__class__.allow_overwrite = True + + if error_models is None: + error_models = ERROR_MODELS.copy() + + if selectors is None: + selectors = SELECTORS.copy() + + config_pars = CommonConfigParams.copy() + + self.reddener = Reddener.build( + dustmap_dir=dustmap_dir, + copy_all_cols=True, + ) + previous_stage = self.reddener + + if blending: + self.unrec_bl = UnrecBlModel.build() + previous_stage = self.unrec_bl + + previous_stage = self.reddener + for key, val in error_models.items(): + error_model_class = ceci.PipelineStage.get_stage(val['ErrorModel'], val['Module']) + the_error_model = error_model_class.make_and_connect( + name=f'error_model_{key}', + connections=dict(input=previous_stage.io.output), + hdf5_groupname='', + ) + self.add_stage(the_error_model) + previous_stage = the_error_model + + dereddener_errors = Dereddener.make_and_connect( + name=f"deredden_{key}", + dustmap_dir=dustmap_dir, + connections=dict(input=previous_stage.io.output), + copy_all_cols=True, + ) + self.add_stage(dereddener_errors) + previous_stage = dereddener_errors + + for key, val in selectors.items(): + the_class = ceci.PipelineStage.get_stage(val['Select'], val['Module']) + the_selector = the_class.make_and_connect( + name=f'select_{key}', + **config_pars, + ) + self.add_stage(the_selector) diff --git a/tests/astro_tools/test_pipline.py b/tests/astro_tools/test_pipline.py index 75efe47..17262ae 100644 --- a/tests/astro_tools/test_pipline.py +++ b/tests/astro_tools/test_pipline.py @@ -9,6 +9,7 @@ 'rail.pipelines.degradation.apply_phot_errors.ApplyPhotErrorsPipeline', 'rail.pipelines.degradation.blending.BlendingPipeline', 'rail.pipelines.degradation.spectroscopic_selection_pipeline.SpectroscopicSelectionPipeline', + 'rail.pipelines.degradation.truth_to_obserbed.TruthToObservedPipeline', ] ) def test_build_and_read_pipeline(pipeline_class): From 623a77a08d7b5eff4b0858759b2bc4fbd3e7ba0a Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Mon, 2 Dec 2024 12:55:05 -0800 Subject: [PATCH 2/4] Added truth_to_observed --- src/rail/pipelines/degradation/truth_to_observed.py | 12 ++++++------ tests/astro_tools/test_pipline.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/rail/pipelines/degradation/truth_to_observed.py b/src/rail/pipelines/degradation/truth_to_observed.py index 7bfd821..8d3d944 100644 --- a/src/rail/pipelines/degradation/truth_to_observed.py +++ b/src/rail/pipelines/degradation/truth_to_observed.py @@ -19,7 +19,7 @@ dustmap_dir = os.path.expandvars("${PZ_DUSTMAP_DIR}") -from .spectroscopic_selection_pipeline.py import SELECTORS, CommonConfigParams +from .spectroscopic_selection_pipeline import SELECTORS, CommonConfigParams from .apply_phot_errors import ERROR_MODELS @@ -27,7 +27,7 @@ class TruthToObservedPipeline(RailPipeline): default_input_dict = dict(input='dummy.in') - def __init__(self, error_models=None, selectors=None, blending=False)): + def __init__(self, error_models=None, selectors=None, blending=False): RailPipeline.__init__(self) DS = RailStage.data_store @@ -51,7 +51,6 @@ def __init__(self, error_models=None, selectors=None, blending=False)): self.unrec_bl = UnrecBlModel.build() previous_stage = self.unrec_bl - previous_stage = self.reddener for key, val in error_models.items(): error_model_class = ceci.PipelineStage.get_stage(val['ErrorModel'], val['Module']) the_error_model = error_model_class.make_and_connect( @@ -71,10 +70,11 @@ def __init__(self, error_models=None, selectors=None, blending=False)): self.add_stage(dereddener_errors) previous_stage = dereddener_errors - for key, val in selectors.items(): - the_class = ceci.PipelineStage.get_stage(val['Select'], val['Module']) + for key2, val2 in selectors.items(): + the_class = ceci.PipelineStage.get_stage(val2['Select'], val2['Module']) the_selector = the_class.make_and_connect( - name=f'select_{key}', + name=f'select_{key}_{key2}', + connections=dict(input=previous_stage.io.output), **config_pars, ) self.add_stage(the_selector) diff --git a/tests/astro_tools/test_pipline.py b/tests/astro_tools/test_pipline.py index 17262ae..b1f64bc 100644 --- a/tests/astro_tools/test_pipline.py +++ b/tests/astro_tools/test_pipline.py @@ -9,7 +9,7 @@ 'rail.pipelines.degradation.apply_phot_errors.ApplyPhotErrorsPipeline', 'rail.pipelines.degradation.blending.BlendingPipeline', 'rail.pipelines.degradation.spectroscopic_selection_pipeline.SpectroscopicSelectionPipeline', - 'rail.pipelines.degradation.truth_to_obserbed.TruthToObservedPipeline', + 'rail.pipelines.degradation.truth_to_observed.TruthToObservedPipeline', ] ) def test_build_and_read_pipeline(pipeline_class): From 22ef5714200df07414087671633bba8713f304f1 Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Mon, 2 Dec 2024 14:35:57 -0800 Subject: [PATCH 3/4] fixes to unit tests --- .../pipelines/degradation/truth_to_observed.py | 4 +++- tests/astro_tools/test_pipline.py | 14 +++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/rail/pipelines/degradation/truth_to_observed.py b/src/rail/pipelines/degradation/truth_to_observed.py index 8d3d944..89fbee4 100644 --- a/src/rail/pipelines/degradation/truth_to_observed.py +++ b/src/rail/pipelines/degradation/truth_to_observed.py @@ -14,11 +14,13 @@ from rail.core.utils import RAILDIR -if 'PZ_DUSTMAP_DIR' not in os.environ: +if 'PZ_DUSTMAP_DIR' not in os.environ: # pragma: no cover os.environ['PZ_DUSTMAP_DIR'] = '.' dustmap_dir = os.path.expandvars("${PZ_DUSTMAP_DIR}") +from rail.creation.degraders.unrec_bl_model import UnrecBlModel + from .spectroscopic_selection_pipeline import SELECTORS, CommonConfigParams from .apply_phot_errors import ERROR_MODELS diff --git a/tests/astro_tools/test_pipline.py b/tests/astro_tools/test_pipline.py index b1f64bc..61c9910 100644 --- a/tests/astro_tools/test_pipline.py +++ b/tests/astro_tools/test_pipline.py @@ -4,14 +4,14 @@ import pytest @pytest.mark.parametrize( - "pipeline_class", + "pipeline_class, options", [ - 'rail.pipelines.degradation.apply_phot_errors.ApplyPhotErrorsPipeline', - 'rail.pipelines.degradation.blending.BlendingPipeline', - 'rail.pipelines.degradation.spectroscopic_selection_pipeline.SpectroscopicSelectionPipeline', - 'rail.pipelines.degradation.truth_to_observed.TruthToObservedPipeline', + ('rail.pipelines.degradation.apply_phot_errors.ApplyPhotErrorsPipeline', {}), + ('rail.pipelines.degradation.blending.BlendingPipeline', {}), + ('rail.pipelines.degradation.spectroscopic_selection_pipeline.SpectroscopicSelectionPipeline', {}), + ('rail.pipelines.degradation.truth_to_observed.TruthToObservedPipeline', {'blending':True}), ] ) -def test_build_and_read_pipeline(pipeline_class): - build_and_read_pipeline(pipeline_class) +def test_build_and_read_pipeline(pipeline_class, options): + build_and_read_pipeline(pipeline_class, **options) From fd642632f38212a716aa488ca5d6bf95d41ba2c8 Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Mon, 2 Dec 2024 21:11:48 -0800 Subject: [PATCH 4/4] Sid's comments --- src/rail/pipelines/degradation/truth_to_observed.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/rail/pipelines/degradation/truth_to_observed.py b/src/rail/pipelines/degradation/truth_to_observed.py index 89fbee4..a6708f2 100644 --- a/src/rail/pipelines/degradation/truth_to_observed.py +++ b/src/rail/pipelines/degradation/truth_to_observed.py @@ -14,17 +14,18 @@ from rail.core.utils import RAILDIR -if 'PZ_DUSTMAP_DIR' not in os.environ: # pragma: no cover - os.environ['PZ_DUSTMAP_DIR'] = '.' - -dustmap_dir = os.path.expandvars("${PZ_DUSTMAP_DIR}") - from rail.creation.degraders.unrec_bl_model import UnrecBlModel from .spectroscopic_selection_pipeline import SELECTORS, CommonConfigParams from .apply_phot_errors import ERROR_MODELS +if 'PZ_DUSTMAP_DIR' not in os.environ: # pragma: no cover + os.environ['PZ_DUSTMAP_DIR'] = '.' + +dustmap_dir = os.path.expandvars("${PZ_DUSTMAP_DIR}") + + class TruthToObservedPipeline(RailPipeline): default_input_dict = dict(input='dummy.in')