Skip to content

Commit

Permalink
Merge pull request #254 from legacysurvey/ADMblobby
Browse files Browse the repository at this point in the history 
output sweeps file should contain bitmask BRIGHTBLOB
  • Loading branch information
@moustakas
moustakas authored Mar 13, 2019
2 parents f788344 + 50a7cd4 commit b89a292
Showing 1 changed file with 164 additions and 153 deletions.
317 changes: 164 additions & 153 deletions bin/generate-sweep-files.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,17 @@ def filter(brickname, filename, region):
return None, None
else:
raise
# ADM check all the column dtypes match.
if not ns.ignore_errors:
sflds = SWEEP_DTYPE.fields
tflds = objects.dtype.fields
for fld in sflds:
sdt, tdt = sflds[fld][0], tflds[fld][0]
if sdt != tdt:
msg = 'sweeps/Tractor dtypes differ for field '
msg += '{}. Sweeps: {}, Tractor: {}'.format(fld, sdt, tdt)
raise ValueError(msg)

mask = objects['BRICK_PRIMARY'] != 0
objects = objects[mask]
mask = objects['RA'] >= ra1
Expand Down Expand Up @@ -264,156 +275,156 @@ def read_region(brickname, filename, bricksdesc):

SWEEP_DTYPE = np.dtype([
# ('BRICK_PRIMARY', '?'),
('RELEASE', '>i4'),
('BRICKID', '>i4'),
('BRICKNAME', 'S8'),
('OBJID', '>i4'),
('TYPE', 'S4'),
('RA', '>f8'),
('DEC', '>f8'),
('RA_IVAR', '>f4'),
('RELEASE', '>i2'),
('BRICKID', '>i4'),
('BRICKNAME', 'S8'),
('OBJID', '>i4'),
('TYPE', 'S4'),
('RA', '>f8'),
('DEC', '>f8'),
('RA_IVAR', '>f4'),
('DEC_IVAR', '>f4'),
('DCHISQ', '>f4', (5,)),
('DCHISQ', '>f4', (5,)),
('EBV', '>f4'),
# ('TYCHO2INBLOB', '?'),
# ('OUT_OF_BOUNDS', '?'),
# ('DECAM_FLUX', '>f4', (6,)),
# ('WISE_FLUX', '>f4', (4,)),
('FLUX_U', '>f4'),
('FLUX_G', '>f4'),
('FLUX_R', '>f4'),
('FLUX_I', '>f4'),
('FLUX_Z', '>f4'),
('FLUX_Y', '>f4'),
('FLUX_W1', '>f4'),
('FLUX_W2', '>f4'),
('FLUX_W3', '>f4'),
('FLUX_W4', '>f4'),
# ('DECAM_FLUX_IVAR', '>f4', (6,)),
# ('WISE_FLUX_IVAR', '>f4', (4,)),
('FLUX_IVAR_U', '>f4'),
('FLUX_IVAR_G', '>f4'),
('FLUX_IVAR_R', '>f4'),
('FLUX_IVAR_I', '>f4'),
('FLUX_IVAR_Z', '>f4'),
('FLUX_IVAR_Y', '>f4'),
('FLUX_IVAR_W1', '>f4'),
('FLUX_IVAR_W2', '>f4'),
('FLUX_IVAR_W3', '>f4'),
('FLUX_IVAR_W4', '>f4'),
# ('DECAM_MW_TRANSMISSION', '>f4', (6,)),
# ('WISE_MW_TRANSMISSION', '>f4', (4,)),
('MW_TRANSMISSION_U', '>f4'),
('MW_TRANSMISSION_G', '>f4'),
('MW_TRANSMISSION_R', '>f4'),
('MW_TRANSMISSION_I', '>f4'),
('MW_TRANSMISSION_Z', '>f4'),
('MW_TRANSMISSION_Y', '>f4'),
('MW_TRANSMISSION_W1', '>f4'),
('MW_TRANSMISSION_W2', '>f4'),
('MW_TRANSMISSION_W3', '>f4'),
('MW_TRANSMISSION_W4', '>f4'),
# ('TYCHO2INBLOB', '?'),
# ('OUT_OF_BOUNDS', '?'),
# ('DECAM_FLUX', '>f4', (6,)),
# ('WISE_FLUX', '>f4', (4,)),
('FLUX_U', '>f4'),
('FLUX_G', '>f4'),
('FLUX_R', '>f4'),
('FLUX_I', '>f4'),
('FLUX_Z', '>f4'),
('FLUX_Y', '>f4'),
('FLUX_W1', '>f4'),
('FLUX_W2', '>f4'),
('FLUX_W3', '>f4'),
('FLUX_W4', '>f4'),
# ('DECAM_FLUX_IVAR', '>f4', (6,)),
# ('WISE_FLUX_IVAR', '>f4', (4,)),
('FLUX_IVAR_U', '>f4'),
('FLUX_IVAR_G', '>f4'),
('FLUX_IVAR_R', '>f4'),
('FLUX_IVAR_I', '>f4'),
('FLUX_IVAR_Z', '>f4'),
('FLUX_IVAR_Y', '>f4'),
('FLUX_IVAR_W1', '>f8'),
('FLUX_IVAR_W2', '>f8'),
('FLUX_IVAR_W3', '>f8'),
('FLUX_IVAR_W4', '>f8'),
# ('DECAM_MW_TRANSMISSION', '>f4', (6,)),
# ('WISE_MW_TRANSMISSION', '>f4', (4,)),
('MW_TRANSMISSION_U', '>f4'),
('MW_TRANSMISSION_G', '>f4'),
('MW_TRANSMISSION_R', '>f4'),
('MW_TRANSMISSION_I', '>f4'),
('MW_TRANSMISSION_Z', '>f4'),
('MW_TRANSMISSION_Y', '>f4'),
('MW_TRANSMISSION_W1', '>f4'),
('MW_TRANSMISSION_W2', '>f4'),
('MW_TRANSMISSION_W3', '>f4'),
('MW_TRANSMISSION_W4', '>f4'),
# ('DECAM_NOBS', 'u1', (6,)),
# ('WISE_NOBS', '>i2', (4,)),
('NOBS_U', 'i2'),
('NOBS_G', 'i2'),
('NOBS_R', 'i2'),
('NOBS_I', 'i2'),
('NOBS_Z', 'i2'),
('NOBS_Y', 'i2'),
('NOBS_W1', 'i2'),
('NOBS_W2', 'i2'),
('NOBS_W3', 'i2'),
('NOBS_W4', 'i2'),
# ('DECAM_RCHI2', '>f4', (6,)),
# ('WISE_RCHI2', '>f4', (4,)),
('RCHISQ_U', '>f4'),
('RCHISQ_G', '>f4'),
('RCHISQ_R', '>f4'),
('RCHISQ_I', '>f4'),
('RCHISQ_Z', '>f4'),
('RCHISQ_Y', '>f4'),
('RCHISQ_W1', '>f4'),
('RCHISQ_W2', '>f4'),
('RCHISQ_W3', '>f4'),
('RCHISQ_W4', '>f4'),
# ('DECAM_FRACFLUX', '>f4', (6,)),
# ('WISE_FRACFLUX', '>f4', (4,)),
('FRACFLUX_U', '>f4'),
('FRACFLUX_G', '>f4'),
('FRACFLUX_R', '>f4'),
('FRACFLUX_I', '>f4'),
('FRACFLUX_Z', '>f4'),
('FRACFLUX_Y', '>f4'),
('FRACFLUX_W1', '>f4'),
('FRACFLUX_W2', '>f4'),
('FRACFLUX_W3', '>f4'),
('FRACFLUX_W4', '>f4'),
# ('DECAM_FRACMASKED', '>f4', (6,)),
('FRACMASKED_U', '>f4'),
('FRACMASKED_G', '>f4'),
('FRACMASKED_R', '>f4'),
('FRACMASKED_I', '>f4'),
('FRACMASKED_Z', '>f4'),
('FRACMASKED_Y', '>f4'),
# ('WISE_NOBS', '>i2', (4,)),
('NOBS_U', '>i2'),
('NOBS_G', '>i2'),
('NOBS_R', '>i2'),
('NOBS_I', '>i2'),
('NOBS_Z', '>i2'),
('NOBS_Y', '>i2'),
('NOBS_W1', '>i2'),
('NOBS_W2', '>i2'),
('NOBS_W3', '>i2'),
('NOBS_W4', '>i2'),
# ('DECAM_RCHI2', '>f4', (6,)),
# ('WISE_RCHI2', '>f4', (4,)),
('RCHISQ_U', '>f4'),
('RCHISQ_G', '>f4'),
('RCHISQ_R', '>f4'),
('RCHISQ_I', '>f4'),
('RCHISQ_Z', '>f4'),
('RCHISQ_Y', '>f4'),
('RCHISQ_W1', '>f4'),
('RCHISQ_W2', '>f4'),
('RCHISQ_W3', '>f4'),
('RCHISQ_W4', '>f4'),
# ('DECAM_FRACFLUX', '>f4', (6,)),
# ('WISE_FRACFLUX', '>f4', (4,)),
('FRACFLUX_U', '>f4'),
('FRACFLUX_G', '>f4'),
('FRACFLUX_R', '>f4'),
('FRACFLUX_I', '>f4'),
('FRACFLUX_Z', '>f4'),
('FRACFLUX_Y', '>f4'),
('FRACFLUX_W1', '>f4'),
('FRACFLUX_W2', '>f4'),
('FRACFLUX_W3', '>f4'),
('FRACFLUX_W4', '>f4'),
# ('DECAM_FRACMASKED', '>f4', (6,)),
('FRACMASKED_U', '>f4'),
('FRACMASKED_G', '>f4'),
('FRACMASKED_R', '>f4'),
('FRACMASKED_I', '>f4'),
('FRACMASKED_Z', '>f4'),
('FRACMASKED_Y', '>f4'),
# ('DECAM_FRACIN', '>f4', (6,)),
('FRACIN_U', '>f4'),
('FRACIN_G', '>f4'),
('FRACIN_R', '>f4'),
('FRACIN_I', '>f4'),
('FRACIN_Z', '>f4'),
('FRACIN_Y', '>f4'),
# ('DECAM_ANYMASK', '>i2', (6,)),
('ANYMASK_U', '>i2'),
('ANYMASK_G', '>i2'),
('ANYMASK_R', '>i2'),
('ANYMASK_I', '>i2'),
('ANYMASK_Z', '>i2'),
('ANYMASK_Y', '>i2'),
# ('DECAM_ALLMASK', '>i2', (6,)),
('ALLMASK_U', '>i2'),
('ALLMASK_G', '>i2'),
('ALLMASK_R', '>i2'),
('ALLMASK_I', '>i2'),
('ALLMASK_Z', '>i2'),
('ALLMASK_Y', '>i2'),
('WISEMASK_W1', 'u1'),
('FRACIN_U', '>f4'),
('FRACIN_G', '>f4'),
('FRACIN_R', '>f4'),
('FRACIN_I', '>f4'),
('FRACIN_Z', '>f4'),
('FRACIN_Y', '>f4'),
# ('DECAM_ANYMASK', '>i2', (6,)),
('ANYMASK_U', '>i2'),
('ANYMASK_G', '>i2'),
('ANYMASK_R', '>i2'),
('ANYMASK_I', '>i2'),
('ANYMASK_Z', '>i2'),
('ANYMASK_Y', '>i2'),
# ('DECAM_ALLMASK', '>i2', (6,)),
('ALLMASK_U', '>i2'),
('ALLMASK_G', '>i2'),
('ALLMASK_R', '>i2'),
('ALLMASK_I', '>i2'),
('ALLMASK_Z', '>i2'),
('ALLMASK_Y', '>i2'),
('WISEMASK_W1', 'u1'),
('WISEMASK_W2', 'u1'),
# ('DECAM_PSFSIZE', '>f4', (6,)),
('PSFSIZE_U', '>f4'),
('PSFSIZE_G', '>f4'),
('PSFSIZE_R', '>f4'),
('PSFSIZE_I', '>f4'),
('PSFSIZE_Z', '>f4'),
('PSFSIZE_Y', '>f4'),
# ('DECAM_DEPTH', '>f4', (6,)),
('PSFDEPTH_U', '>f4'),
('PSFDEPTH_G', '>f4'),
('PSFDEPTH_R', '>f4'),
('PSFDEPTH_I', '>f4'),
('PSFDEPTH_Z', '>f4'),
('PSFDEPTH_Y', '>f4'),
# ('DECAM_GALDEPTH', '>f4', (6,)),
('GALDEPTH_U', '>f4'),
('GALDEPTH_G', '>f4'),
('GALDEPTH_R', '>f4'),
('GALDEPTH_I', '>f4'),
('GALDEPTH_Z', '>f4'),
('GALDEPTH_Y', '>f4'),
('WISE_COADD_ID', 'S8'),
('FRACDEV', '>f4'),
('FRACDEV_IVAR', '>f4'),
('SHAPEDEV_R', '>f4'),
('SHAPEDEV_R_IVAR', '>f4'),
('SHAPEDEV_E1', '>f4'),
('SHAPEDEV_E1_IVAR', '>f4'),
('SHAPEDEV_E2', '>f4'),
('SHAPEDEV_E2_IVAR', '>f4'),
('SHAPEEXP_R', '>f4'),
('SHAPEEXP_R_IVAR', '>f4'),
('SHAPEEXP_E1', '>f4'),
('SHAPEEXP_E1_IVAR', '>f4'),
('SHAPEEXP_E2', '>f4'),
# ('DECAM_PSFSIZE', '>f4', (6,)),
('PSFSIZE_U', '>f4'),
('PSFSIZE_G', '>f4'),
('PSFSIZE_R', '>f4'),
('PSFSIZE_I', '>f4'),
('PSFSIZE_Z', '>f4'),
('PSFSIZE_Y', '>f4'),
# ('DECAM_DEPTH', '>f4', (6,)),
('PSFDEPTH_U', '>f4'),
('PSFDEPTH_G', '>f4'),
('PSFDEPTH_R', '>f4'),
('PSFDEPTH_I', '>f4'),
('PSFDEPTH_Z', '>f4'),
('PSFDEPTH_Y', '>f4'),
# ('DECAM_GALDEPTH', '>f4', (6,)),
('GALDEPTH_U', '>f4'),
('GALDEPTH_G', '>f4'),
('GALDEPTH_R', '>f4'),
('GALDEPTH_I', '>f4'),
('GALDEPTH_Z', '>f4'),
('GALDEPTH_Y', '>f4'),
('WISE_COADD_ID', 'S8'),
('FRACDEV', '>f4'),
('FRACDEV_IVAR', '>f4'),
('SHAPEDEV_R', '>f4'),
('SHAPEDEV_R_IVAR', '>f4'),
('SHAPEDEV_E1', '>f4'),
('SHAPEDEV_E1_IVAR', '>f4'),
('SHAPEDEV_E2', '>f4'),
('SHAPEDEV_E2_IVAR', '>f4'),
('SHAPEEXP_R', '>f4'),
('SHAPEEXP_R_IVAR', '>f4'),
('SHAPEEXP_E1', '>f4'),
('SHAPEEXP_E1_IVAR', '>f4'),
('SHAPEEXP_E2', '>f4'),
('SHAPEEXP_E2_IVAR', '>f4'),
('FIBERFLUX_U', '>f4'),
('FIBERFLUX_G', '>f4'),
Expand Down Expand Up @@ -442,12 +453,12 @@ def read_region(brickname, filename, bricksdesc):
('PMRA_IVAR', '>f4'),
('PMDEC', '>f4'),
('PMDEC_IVAR', '>f4'),
('BRIGHTSTARINBLOB', '>b1')]
('BRIGHTBLOB', '>i2')]
)

def parse_args():
ap = argparse.ArgumentParser(
description="""Create Sweep files for DECALS.
description="""Create Sweep files for DECALS.
This tool ensures each sweep file contains roughly '-n' objects. HDF5 and FITS formats are supported.
Columns contained in a sweep file are:
Expand All @@ -466,23 +477,23 @@ def parse_args():
ap.add_argument('-F', "--filelist", default=None,
help="list of tractor brickfiles to use; this will avoid expensive walking of the path.")

ap.add_argument('-d', "--bricksdesc", default=None,
ap.add_argument('-d', "--bricksdesc", default=None,
help="location of decals-bricks.fits, speeds up the scanning")

ap.add_argument('-v', "--verbose", action='store_true')
ap.add_argument('-I', "--ignore-errors", action='store_true')

ap.add_argument('-S', "--schema", choices=['blocks', 'dec', 'ra'],
default='blocks',
ap.add_argument('-S', "--schema", choices=['blocks', 'dec', 'ra'],
default='blocks',
help="""Decomposition schema. Still being tuned. """)

ap.add_argument('-b', "--bricklist",
help="""Filename with list of bricknames to include.
ap.add_argument('-b', "--bricklist",
help="""Filename with list of bricknames to include.
If not set, all bricks in src are included, sorted by brickname.
""")

ap.add_argument("--numproc", type=int, default=None,
help="""Number of concurrent processes to use. 0 for sequential execution.
help="""Number of concurrent processes to use. 0 for sequential execution.
Default is to use OMP_NUM_THREADS, or the number of cores on the node.""")

return ap.parse_args()
Expand Down

0 comments on commit b89a292

Please sign in to comment.