From 71556b4fdb037439fb924224cdeaaf7025c99dae Mon Sep 17 00:00:00 2001 From: Yusra AlSayyad Date: Thu, 13 Feb 2020 16:42:15 -0600 Subject: [PATCH 1/2] Enable per-band flags in transformObject by splitting the `flags` key into two: * `flags` which will expanded out per band from meas * `refFlags` will be taked from ref --- python/lsst/pipe/tasks/functors.py | 6 +++++- python/lsst/pipe/tasks/postprocess.py | 31 +++++++++++++++++---------- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/python/lsst/pipe/tasks/functors.py b/python/lsst/pipe/tasks/functors.py index d6f56a860..d18994ba3 100644 --- a/python/lsst/pipe/tasks/functors.py +++ b/python/lsst/pipe/tasks/functors.py @@ -328,9 +328,13 @@ def from_yaml(cls, translationDefinition, **kwargs): else: renameRules = None + if 'refFlags' in translationDefinition: + for flag in translationDefinition['refFlags']: + funcs[cls.renameCol(flag, renameRules)] = Column(flag, dataset='ref') + if 'flags' in translationDefinition: for flag in translationDefinition['flags']: - funcs[cls.renameCol(flag, renameRules)] = Column(flag, dataset='ref') + funcs[cls.renameCol(flag, renameRules)] = Column(flag, dataset='meas') return cls(funcs, **kwargs) diff --git a/python/lsst/pipe/tasks/postprocess.py b/python/lsst/pipe/tasks/postprocess.py index 1ef42ed5b..478581a06 100644 --- a/python/lsst/pipe/tasks/postprocess.py +++ b/python/lsst/pipe/tasks/postprocess.py @@ -221,8 +221,8 @@ class PostprocessAnalysis(object): functor collection, since the `filt` keyword argument of this object triggers an overwrite of the `filt` property for all functors in the collection. - This object also allows a list of flags to be passed, and defines a set of default - flags that are always included even if not requested. + This object also allows a list of refFlags to be passed, and defines a set of default + refFlags that are always included even if not requested. If a list of `ParquetTable` object is passed, rather than a single one, then the calculations will be mapped over all the input catalogs. In principle, it should @@ -247,20 +247,26 @@ class PostprocessAnalysis(object): of the provided functors. flags : `list` (optional) - List of flags to include in output table. + List of flags (per-band) to include in output table. + + refFlags : `list` (optional) + List of refFlags (only reference band) to include in output table. + + """ - _defaultFlags = ('calib_psf_used', 'detect_isPrimary') + _defaultRefFlags = ('calib_psf_used', 'detect_isPrimary') _defaultFuncs = (('coord_ra', RAColumn()), ('coord_dec', DecColumn())) - def __init__(self, parq, functors, filt=None, flags=None): + def __init__(self, parq, functors, filt=None, flags=None, refFlags=None): self.parq = parq self.functors = functors self.filt = filt - self.flags = list(self._defaultFlags) - if flags is not None: - self.flags += list(flags) + self.flags = list(flags) if flags is not None else [] + self.refFlags = list(self._defaultRefFlags) + if refFlags is not None: + self.refFlags += list(refFlags) self._df = None @@ -272,7 +278,8 @@ def defaultFuncs(self): @property def func(self): additionalFuncs = self.defaultFuncs - additionalFuncs.update({flag: Column(flag) for flag in self.flags}) + additionalFuncs.update({flag: Column(flag, dataset='ref') for flag in self.refFlags}) + additionalFuncs.update({flag: Column(flag, dataset='meas') for flag in self.flags}) if isinstance(self.functors, CompositeFunctor): func = self.functors @@ -364,7 +371,7 @@ class TransformCatalogBaseTask(CmdLineTask): functor: DeconvolvedMoments filt: HSC-G dataset: forced_src - flags: + refFlags: - calib_psfUsed - merge_measurement_i - merge_measurement_r @@ -380,9 +387,11 @@ class TransformCatalogBaseTask(CmdLineTask): and any additional entries for each column other than "functor" or "args" (e.g., `'filt'`, `'dataset'`) are treated as keyword arguments to be passed to the functor initialization. - The "flags" entry is shortcut for a bunch of `Column` functors with the original column and + The "refFlags" entry is shortcut for a bunch of `Column` functors with the original column and taken from the `'ref'` dataset. + The "flags" entry will be expanded out per band. + Note, if `'filter'` is provided as part of the `dataId` when running this task (even though `deepCoadd_obj` does not use `'filter'`), then this will override the `filt` kwargs provided in the YAML file, and the calculations will be done in that filter. From 3e2e8b6e9602af58c28c28aa628ef4d02eb8d6de Mon Sep 17 00:00:00 2001 From: Yusra AlSayyad Date: Thu, 13 Feb 2020 17:38:36 -0600 Subject: [PATCH 2/2] Make CoordColumn more reusable by behaving like other Columns which explode per band by default. RAColumn and DecColumn are now the non-exploding special cases that still produce a single reference band column. Change the behavior of intermediate CoordColumn preferred over adding a second intermediate class in the class hierarchy. --- python/lsst/pipe/tasks/functors.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/python/lsst/pipe/tasks/functors.py b/python/lsst/pipe/tasks/functors.py index d18994ba3..f9708aedf 100644 --- a/python/lsst/pipe/tasks/functors.py +++ b/python/lsst/pipe/tasks/functors.py @@ -449,12 +449,9 @@ class FootprintNPix(Column): class CoordColumn(Column): """Base class for coordinate column, in degrees """ - _allow_difference = False _radians = True - _defaultNoDup = True - def __init__(self, col, calculate=False, **kwargs): - self.calculate = calculate + def __init__(self, col, **kwargs): super().__init__(col, **kwargs) def _func(self, df): @@ -468,6 +465,7 @@ class RAColumn(CoordColumn): """Right Ascension, in degrees """ name = 'RA' + _defaultNoDup = True def __init__(self, **kwargs): super().__init__('coord_ra', **kwargs) @@ -480,6 +478,7 @@ class DecColumn(CoordColumn): """Declination, in degrees """ name = 'Dec' + _defaultNoDup = True def __init__(self, **kwargs): super().__init__('coord_dec', **kwargs)