Skip to content

Commit

Permalink
Merge branch 'tickets/DM-21789'
Browse files Browse the repository at this point in the history
  • Loading branch information
@taranu
taranu committed Sep 15, 2023
2 parents 39803aa + fc6fcfc commit e80f7f7
Show file tree
Hide file tree
Showing 2 changed files with 55 additions and 58 deletions.
98 changes: 47 additions & 51 deletions python/lsst/pipe/tasks/diff_matched_tract_catalog.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@
import numpy as np
import pandas as pd
from scipy.stats import iqr
from typing import Dict, Sequence, Set
from typing import Dict, Sequence


def is_sequence_set(x: Sequence):
Expand Down Expand Up @@ -136,13 +136,17 @@ class MatchedCatalogFluxesConfig(pexConfig.Config):
doc="List of target catalog flux error column names",
)

# this should be an orderedset
@property
def columns_in_ref(self) -> Set[str]:
return {self.column_ref_flux}
def columns_in_ref(self) -> list[str]:
return [self.column_ref_flux]

# this should also be an orderedset
@property
def columns_in_target(self) -> Set[str]:
return set(self.columns_target_flux).union(set(self.columns_target_flux_err))
def columns_in_target(self) -> list[str]:
columns = [col for col in self.columns_target_flux]
columns.extend(col for col in self.columns_target_flux_err if col not in columns)
return columns


class DiffMatchedTractCatalogConfig(
Expand Down Expand Up @@ -171,7 +175,7 @@ class DiffMatchedTractCatalogConfig(
)

@property
def columns_in_ref(self) -> Set[str]:
def columns_in_ref(self) -> list[str]:
columns_all = [self.coord_format.column_ref_coord1, self.coord_format.column_ref_coord2,
self.column_ref_extended]
for column_lists in (
Expand All @@ -186,11 +190,12 @@ def columns_in_ref(self) -> Set[str]:
return set(columns_all)

@property
def columns_in_target(self) -> Set[str]:
def columns_in_target(self) -> list[str]:
columns_all = [self.coord_format.column_target_coord1, self.coord_format.column_target_coord2,
self.column_target_extended]
if self.coord_format.coords_ref_to_convert is not None:
columns_all.extend(self.coord_format.coords_ref_to_convert.values())
columns_all.extend(col for col in self.coord_format.coords_ref_to_convert.values()
if col not in columns_all)
for column_lists in (
(
self.columns_target_coord_err,
Expand All @@ -201,84 +206,75 @@ def columns_in_target(self) -> Set[str]:
(x.columns_in_target for x in self.columns_flux.values()),
):
for column_list in column_lists:
columns_all.extend(column_list)
return set(columns_all)
columns_all.extend(col for col in column_list if col not in columns_all)
return columns_all

columns_flux = pexConfig.ConfigDictField(
doc="Configs for flux columns for each band",
keytype=str,
itemtype=MatchedCatalogFluxesConfig,
doc="Configs for flux columns for each band",
)
columns_ref_copy = pexConfig.ListField(
dtype=str,
default=set(),
doc='Reference table columns to copy to copy into cat_matched',
columns_ref_copy = pexConfig.ListField[str](
doc='Reference table columns to copy into cat_matched',
default=[],
listCheck=is_sequence_set,
)
columns_target_coord_err = pexConfig.ListField(
dtype=str,
listCheck=lambda x: (len(x) == 2) and (x[0] != x[1]),
columns_target_coord_err = pexConfig.ListField[str](
doc='Target table coordinate columns with standard errors (sigma)',
listCheck=lambda x: (len(x) == 2) and (x[0] != x[1]),
)
columns_target_copy = pexConfig.ListField(
dtype=str,
columns_target_copy = pexConfig.ListField[str](
doc='Target table columns to copy into cat_matched',
default=('patch',),
doc='Target table columns to copy to copy into cat_matched',
listCheck=is_sequence_set,
)
columns_target_select_true = pexConfig.ListField(
dtype=str,
default=('detect_isPrimary',),
columns_target_select_true = pexConfig.ListField[str](
doc='Target table columns to require to be True for selecting sources',
default=('detect_isPrimary',),
listCheck=is_sequence_set,
)
columns_target_select_false = pexConfig.ListField(
dtype=str,
default=('merge_peak_sky',),
columns_target_select_false = pexConfig.ListField[str](
doc='Target table columns to require to be False for selecting sources',
default=('merge_peak_sky',),
listCheck=is_sequence_set,
)
coord_format = pexConfig.ConfigField(
dtype=ConvertCatalogCoordinatesConfig,
coord_format = pexConfig.ConfigField[ConvertCatalogCoordinatesConfig](
doc="Configuration for coordinate conversion",
)
extendedness_cut = pexConfig.Field(
extendedness_cut = pexConfig.Field[float](
dtype=float,
default=0.5,
doc='Minimum extendedness for a measured source to be considered extended',
)
mag_num_bins = pexConfig.Field(
mag_num_bins = pexConfig.Field[int](
doc='Number of magnitude bins',
default=15,
dtype=int,
)
mag_brightest_ref = pexConfig.Field(
dtype=float,
default=15,
mag_brightest_ref = pexConfig.Field[float](
doc='Brightest magnitude cutoff for binning',
default=15,
)
mag_ceiling_target = pexConfig.Field(
dtype=float,
mag_ceiling_target = pexConfig.Field[float](
doc='Ceiling (maximum/faint) magnitude for target sources',
default=None,
optional=True,
doc='Ceiling (maximum/faint) magnitude for target sources',
)
mag_faintest_ref = pexConfig.Field(
dtype=float,
default=30,
mag_faintest_ref = pexConfig.Field[float](
doc='Faintest magnitude cutoff for binning',
default=30,
)
mag_zeropoint_ref = pexConfig.Field(
dtype=float,
default=31.4,
mag_zeropoint_ref = pexConfig.Field[float](
doc='Magnitude zeropoint for reference sources',
)
mag_zeropoint_target = pexConfig.Field(
dtype=float,
default=31.4,
)
mag_zeropoint_target = pexConfig.Field[float](
doc='Magnitude zeropoint for target sources',
default=31.4,
)
percentiles = pexConfig.ListField(
dtype=str,
percentiles = pexConfig.ListField[str](
doc='Percentiles to compute for diff/chi values',
# -2, -1, +1, +2 sigma percentiles for normal distribution
default=('2.275', '15.866', '84.134', '97.725'),
doc='Percentiles to compute for diff/chi values',
itemCheck=is_percentile,
listCheck=is_sequence_set,
)
Expand Down Expand Up @@ -649,7 +645,7 @@ def run(
cat_left = cat_target.iloc[matched_row]
has_index_left = cat_left.index.name is not None
cat_right = cat_ref[matched_ref].reset_index()
cat_matched = pd.concat(objs=(cat_left.reset_index(drop=True), cat_right), axis=1)
cat_matched = pd.concat(objs=(cat_left.reset_index(drop=True), cat_right), axis=1, sort=False)
if has_index_left:
cat_matched.index = cat_left.index
cat_matched.columns.values[len(cat_target.columns):] = [f'refcat_{col}' for col in cat_right.columns]
Expand Down
15 changes: 8 additions & 7 deletions python/lsst/pipe/tasks/fit_coadd_multiband.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,22 +141,22 @@ def adjustQuantum(self, inputs, outputs, label, data_id):
"""
# Check which bands are going to be fit
bands_fit, bands_read_only = self.config.get_band_sets()
bands_needed = bands_fit.union(bands_read_only)
bands_needed = bands_fit + [band for band in bands_read_only if band not in bands_fit]

adjusted_inputs = {}
for connection_name, (connection, dataset_refs) in inputs.items():
# Datasets without bands in their dimensions should be fine
if 'band' in connection.dimensions:
datasets_by_band = {dref.dataId['band']: dref for dref in dataset_refs}
if not bands_needed.issubset(datasets_by_band.keys()):
if not set(bands_needed).issubset(datasets_by_band.keys()):
raise pipeBase.NoWorkFound(
f'DatasetRefs={dataset_refs} have data with bands in the'
f' set={set(datasets_by_band.keys())},'
f' which is not a superset of the required bands={bands_needed} defined by'
f' {self.config.__class__}.fit_coadd_multiband='
f'{self.config.fit_coadd_multiband._value.__class__}\'s attributes'
f' bands_fit={bands_fit} and bands_read_only()={bands_read_only}.'
f' Add the required bands={bands_needed.difference(datasets_by_band.keys())}.'
f' Add the required bands={set(bands_needed).difference(datasets_by_band.keys())}.'
)
# Adjust all datasets with band dimensions to include just
# the needed bands, in consistent order.
Expand Down Expand Up @@ -258,7 +258,7 @@ def get_band_sets(self):
except AttributeError:
raise RuntimeError(f'{__class__}.fit_coadd_multiband must have bands_fit attribute') from None
bands_read_only = self.fit_coadd_multiband.bands_read_only()
return set(bands_fit), set(bands_read_only)
return tuple(list({band: None for band in bands}.keys()) for bands in (bands_fit, bands_read_only))


class CoaddMultibandFitTask(pipeBase.PipelineTask):
Expand Down Expand Up @@ -287,8 +287,8 @@ def runQuantum(self, butlerQC, inputRefs, outputRefs):
]
dataIds = set(cats).union(set(exps))
models_scarlet = inputs["models_scarlet"]
catexps = [None]*len(dataIds)
for idx, dataId in enumerate(dataIds):
catexps = {}
for dataId in dataIds:
catalog = cats[dataId]
exposure = exps[dataId]
models_scarlet.updateCatalogFootprints(
Expand All @@ -299,10 +299,11 @@ def runQuantum(self, butlerQC, inputRefs, outputRefs):
removeScarletData=True,
updateFluxColumns=False,
)
catexps[idx] = CatalogExposureInputs(
catexps[dataId['band']] = CatalogExposureInputs(
catalog=catalog, exposure=exposure, table_psf_fits=models_psf[dataId],
dataId=dataId, id_tract_patch=id_tp,
)
catexps = [catexps[band] for band in self.config.get_band_sets()[0]]
outputs = self.run(catexps=catexps, cat_ref=inputs['cat_ref'])
butlerQC.put(outputs, outputRefs)

Expand Down

0 comments on commit e80f7f7

Please sign in to comment.