diff --git a/eastlake/pipeline.py b/eastlake/pipeline.py index 7d88c95..42274a3 100644 --- a/eastlake/pipeline.py +++ b/eastlake/pipeline.py @@ -18,6 +18,7 @@ MEDSRunner, TrueDetectionRunner, DeleteImages, + DeleteSources, DeleteMeds, NewishMetcalRunner, CoaddNwgintRunner, @@ -47,6 +48,7 @@ ('metadetect', MetadetectRunner), ('stash_prep', StashPrep), ('balrog', BalrogRunner), + ('delete_sources', DeleteSources), ]) STEP_IS_GALSIM = set(["galsim"]) diff --git a/eastlake/steps/__init__.py b/eastlake/steps/__init__.py index eb3a997..5dd21cd 100644 --- a/eastlake/steps/__init__.py +++ b/eastlake/steps/__init__.py @@ -11,3 +11,4 @@ from .mdet import MetadetectRunner # noqa from .stash_prep import StashPrep # noqa from .balrog import BalrogRunner # noqa +from .delete_sources import DeleteSources # noqa diff --git a/eastlake/steps/delete_meds.py b/eastlake/steps/delete_meds.py index c1c8ee6..f0d5c58 100644 --- a/eastlake/steps/delete_meds.py +++ b/eastlake/steps/delete_meds.py @@ -32,4 +32,9 @@ def execute(self, stash, new_params=None): self.logger.error("removing meds file %s" % m) os.remove(m) + meds_files = stash.get_filepaths("pizza_cutter_meds_files", tilename) + for m in meds_files: + self.logger.error("removing pizza-cutter meds file %s" % m) + os.remove(m) + return 0, stash diff --git a/eastlake/steps/delete_sources.py b/eastlake/steps/delete_sources.py new file mode 100644 index 0000000..0247e4b --- /dev/null +++ b/eastlake/steps/delete_sources.py @@ -0,0 +1,103 @@ +from __future__ import print_function, absolute_import +import os + +from ..step import Step + + +class DeleteSources(Step): + """ + Pipeline for deleteing all of the source data for a tile. + """ + def __init__(self, config, base_dir, name="delete_sources", + logger=None, verbosity=0, log_file=None): + + # name for this step + super(DeleteSources, self).__init__( + config, base_dir, name=name, logger=logger, verbosity=verbosity, + log_file=log_file) + + if "save_tilenames" not in self.config: + self.config["save_tilenames"] = [] + + def execute(self, stash, new_params=None): + + tilenames = stash["tilenames"] + for tilename in tilenames: + if tilename in self.config["save_tilenames"]: + continue + + self.logger.error("deleting coadd images for tile %s" % tilename) + # First check for a detection coadd, weight, mask and seg + for key in ["det_image_file", "det_weight_file", "det_mask_file"]: + filename = stash.get_filepaths(key, tilename, keyerror=False) + if filename is not None: + if os.path.isfile(filename): + self.logger.error("removing file %s" % filename) + os.remove(filename) + else: + self.logger.error("file %s not found" % filename) + else: + self.logger.error("key %s not present" % key) + + # Now the per-band coadds + for band in stash["bands"]: + coadd_file = stash.get_filepaths( + "coadd_file", tilename, band=band, + keyerror=False, + ) + if (coadd_file is not None): + if os.path.isfile(coadd_file): + os.remove(coadd_file) + + # Also check for seg file + if self.config["delete_seg"]: + seg_file = stash.get_filepaths( + "seg_file", tilename, band=band, + keyerror=False, + ) + if (seg_file is not None): + if os.path.isfile(seg_file): + self.logger.error("removing file %s" % seg_file) + + # Also check for bkg and bkg-rms files + bkg_file = coadd_file.replace(".fits", "bkg.fits") + if os.path.isfile(bkg_file): + os.remove(bkg_file) + bkg_rms_file = coadd_file.replace(".fits", "bkg-rms.fits") + if os.path.isfile(bkg_rms_file): + os.remove(bkg_rms_file) + + self.logger.error("deleting se images for tile %s" % tilename) + for band in stash["bands"]: + img_files = stash.get_filepaths( + "img_files", tilename, band=band, + keyerror=False) + if (img_files is not None): + for f in img_files: + if os.path.isfile(f): + os.remove(f) + + self.logger.error("deleting se nwgint images for tile %s" % tilename) + for band in stash["bands"]: + img_files = stash.get_filepaths( + "coadd_nwgint_img_files", tilename, band=band, + keyerror=False) + if (img_files is not None): + for f in img_files: + if os.path.isfile(f): + os.remove(f) + + self.logger.error("deleting as much as we can for tile %s" % tilename) + for band in stash["bands"]: + pyml = stash.get_output_pizza_cutter_yaml(tilename, band) + for k, v in pyml.items(): + if isinstance(v, str) and os.path.isfile(v): + os.remove(v) + + if k == "srcinfo": + for srci in pyml["srcinfo"]: + for _v in srci.items(): + if isinstance(_v, str) and os.path.isfile(_v): + os.remove(_v) + + return 0, stash