diff --git a/python/lsst/pipe/tasks/diff_matched_tract_catalog.py b/python/lsst/pipe/tasks/diff_matched_tract_catalog.py index 3da7d8d61..a84827751 100644 --- a/python/lsst/pipe/tasks/diff_matched_tract_catalog.py +++ b/python/lsst/pipe/tasks/diff_matched_tract_catalog.py @@ -43,7 +43,7 @@ import numpy as np import pandas as pd from scipy.stats import iqr -from typing import Dict, Sequence, Set +from typing import Dict, Sequence def is_sequence_set(x: Sequence): @@ -136,13 +136,17 @@ class MatchedCatalogFluxesConfig(pexConfig.Config): doc="List of target catalog flux error column names", ) + # this should be an orderedset @property - def columns_in_ref(self) -> Set[str]: - return {self.column_ref_flux} + def columns_in_ref(self) -> list[str]: + return [self.column_ref_flux] + # this should also be an orderedset @property - def columns_in_target(self) -> Set[str]: - return set(self.columns_target_flux).union(set(self.columns_target_flux_err)) + def columns_in_target(self) -> list[str]: + columns = [col for col in self.columns_target_flux] + columns.extend(col for col in self.columns_target_flux_err if col not in columns) + return columns class DiffMatchedTractCatalogConfig( @@ -171,7 +175,7 @@ class DiffMatchedTractCatalogConfig( ) @property - def columns_in_ref(self) -> Set[str]: + def columns_in_ref(self) -> list[str]: columns_all = [self.coord_format.column_ref_coord1, self.coord_format.column_ref_coord2, self.column_ref_extended] for column_lists in ( @@ -186,11 +190,12 @@ def columns_in_ref(self) -> Set[str]: return set(columns_all) @property - def columns_in_target(self) -> Set[str]: + def columns_in_target(self) -> list[str]: columns_all = [self.coord_format.column_target_coord1, self.coord_format.column_target_coord2, self.column_target_extended] if self.coord_format.coords_ref_to_convert is not None: - columns_all.extend(self.coord_format.coords_ref_to_convert.values()) + columns_all.extend(col for col in self.coord_format.coords_ref_to_convert.values() + if col not in columns_all) for column_lists in ( ( self.columns_target_coord_err, @@ -201,84 +206,75 @@ def columns_in_target(self) -> Set[str]: (x.columns_in_target for x in self.columns_flux.values()), ): for column_list in column_lists: - columns_all.extend(column_list) - return set(columns_all) + columns_all.extend(col for col in column_list if col not in columns_all) + return columns_all columns_flux = pexConfig.ConfigDictField( + doc="Configs for flux columns for each band", keytype=str, itemtype=MatchedCatalogFluxesConfig, - doc="Configs for flux columns for each band", ) - columns_ref_copy = pexConfig.ListField( - dtype=str, - default=set(), - doc='Reference table columns to copy to copy into cat_matched', + columns_ref_copy = pexConfig.ListField[str]( + doc='Reference table columns to copy into cat_matched', + default=[], + listCheck=is_sequence_set, ) - columns_target_coord_err = pexConfig.ListField( - dtype=str, - listCheck=lambda x: (len(x) == 2) and (x[0] != x[1]), + columns_target_coord_err = pexConfig.ListField[str]( doc='Target table coordinate columns with standard errors (sigma)', + listCheck=lambda x: (len(x) == 2) and (x[0] != x[1]), ) - columns_target_copy = pexConfig.ListField( - dtype=str, + columns_target_copy = pexConfig.ListField[str]( + doc='Target table columns to copy into cat_matched', default=('patch',), - doc='Target table columns to copy to copy into cat_matched', + listCheck=is_sequence_set, ) - columns_target_select_true = pexConfig.ListField( - dtype=str, - default=('detect_isPrimary',), + columns_target_select_true = pexConfig.ListField[str]( doc='Target table columns to require to be True for selecting sources', + default=('detect_isPrimary',), + listCheck=is_sequence_set, ) - columns_target_select_false = pexConfig.ListField( - dtype=str, - default=('merge_peak_sky',), + columns_target_select_false = pexConfig.ListField[str]( doc='Target table columns to require to be False for selecting sources', + default=('merge_peak_sky',), + listCheck=is_sequence_set, ) - coord_format = pexConfig.ConfigField( - dtype=ConvertCatalogCoordinatesConfig, + coord_format = pexConfig.ConfigField[ConvertCatalogCoordinatesConfig]( doc="Configuration for coordinate conversion", ) - extendedness_cut = pexConfig.Field( + extendedness_cut = pexConfig.Field[float]( dtype=float, default=0.5, doc='Minimum extendedness for a measured source to be considered extended', ) - mag_num_bins = pexConfig.Field( + mag_num_bins = pexConfig.Field[int]( doc='Number of magnitude bins', default=15, - dtype=int, ) - mag_brightest_ref = pexConfig.Field( - dtype=float, - default=15, + mag_brightest_ref = pexConfig.Field[float]( doc='Brightest magnitude cutoff for binning', + default=15, ) - mag_ceiling_target = pexConfig.Field( - dtype=float, + mag_ceiling_target = pexConfig.Field[float]( + doc='Ceiling (maximum/faint) magnitude for target sources', default=None, optional=True, - doc='Ceiling (maximum/faint) magnitude for target sources', ) - mag_faintest_ref = pexConfig.Field( - dtype=float, - default=30, + mag_faintest_ref = pexConfig.Field[float]( doc='Faintest magnitude cutoff for binning', + default=30, ) - mag_zeropoint_ref = pexConfig.Field( - dtype=float, - default=31.4, + mag_zeropoint_ref = pexConfig.Field[float]( doc='Magnitude zeropoint for reference sources', - ) - mag_zeropoint_target = pexConfig.Field( - dtype=float, default=31.4, + ) + mag_zeropoint_target = pexConfig.Field[float]( doc='Magnitude zeropoint for target sources', + default=31.4, ) - percentiles = pexConfig.ListField( - dtype=str, + percentiles = pexConfig.ListField[str]( + doc='Percentiles to compute for diff/chi values', # -2, -1, +1, +2 sigma percentiles for normal distribution default=('2.275', '15.866', '84.134', '97.725'), - doc='Percentiles to compute for diff/chi values', itemCheck=is_percentile, listCheck=is_sequence_set, ) @@ -649,7 +645,7 @@ def run( cat_left = cat_target.iloc[matched_row] has_index_left = cat_left.index.name is not None cat_right = cat_ref[matched_ref].reset_index() - cat_matched = pd.concat(objs=(cat_left.reset_index(drop=True), cat_right), axis=1) + cat_matched = pd.concat(objs=(cat_left.reset_index(drop=True), cat_right), axis=1, sort=False) if has_index_left: cat_matched.index = cat_left.index cat_matched.columns.values[len(cat_target.columns):] = [f'refcat_{col}' for col in cat_right.columns] diff --git a/python/lsst/pipe/tasks/fit_coadd_multiband.py b/python/lsst/pipe/tasks/fit_coadd_multiband.py index 95d1c74a2..345eb057f 100644 --- a/python/lsst/pipe/tasks/fit_coadd_multiband.py +++ b/python/lsst/pipe/tasks/fit_coadd_multiband.py @@ -141,14 +141,14 @@ def adjustQuantum(self, inputs, outputs, label, data_id): """ # Check which bands are going to be fit bands_fit, bands_read_only = self.config.get_band_sets() - bands_needed = bands_fit.union(bands_read_only) + bands_needed = bands_fit + [band for band in bands_read_only if band not in bands_fit] adjusted_inputs = {} for connection_name, (connection, dataset_refs) in inputs.items(): # Datasets without bands in their dimensions should be fine if 'band' in connection.dimensions: datasets_by_band = {dref.dataId['band']: dref for dref in dataset_refs} - if not bands_needed.issubset(datasets_by_band.keys()): + if not set(bands_needed).issubset(datasets_by_band.keys()): raise pipeBase.NoWorkFound( f'DatasetRefs={dataset_refs} have data with bands in the' f' set={set(datasets_by_band.keys())},' @@ -156,7 +156,7 @@ def adjustQuantum(self, inputs, outputs, label, data_id): f' {self.config.__class__}.fit_coadd_multiband=' f'{self.config.fit_coadd_multiband._value.__class__}\'s attributes' f' bands_fit={bands_fit} and bands_read_only()={bands_read_only}.' - f' Add the required bands={bands_needed.difference(datasets_by_band.keys())}.' + f' Add the required bands={set(bands_needed).difference(datasets_by_band.keys())}.' ) # Adjust all datasets with band dimensions to include just # the needed bands, in consistent order. @@ -258,7 +258,7 @@ def get_band_sets(self): except AttributeError: raise RuntimeError(f'{__class__}.fit_coadd_multiband must have bands_fit attribute') from None bands_read_only = self.fit_coadd_multiband.bands_read_only() - return set(bands_fit), set(bands_read_only) + return tuple(list({band: None for band in bands}.keys()) for bands in (bands_fit, bands_read_only)) class CoaddMultibandFitTask(pipeBase.PipelineTask): @@ -287,8 +287,8 @@ def runQuantum(self, butlerQC, inputRefs, outputRefs): ] dataIds = set(cats).union(set(exps)) models_scarlet = inputs["models_scarlet"] - catexps = [None]*len(dataIds) - for idx, dataId in enumerate(dataIds): + catexps = {} + for dataId in dataIds: catalog = cats[dataId] exposure = exps[dataId] models_scarlet.updateCatalogFootprints( @@ -299,10 +299,11 @@ def runQuantum(self, butlerQC, inputRefs, outputRefs): removeScarletData=True, updateFluxColumns=False, ) - catexps[idx] = CatalogExposureInputs( + catexps[dataId['band']] = CatalogExposureInputs( catalog=catalog, exposure=exposure, table_psf_fits=models_psf[dataId], dataId=dataId, id_tract_patch=id_tp, ) + catexps = [catexps[band] for band in self.config.get_band_sets()[0]] outputs = self.run(catexps=catexps, cat_ref=inputs['cat_ref']) butlerQC.put(outputs, outputRefs)