From 809f50b759f941e691762c2d6d4d298fce392fc2 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Tue, 12 Mar 2019 12:06:20 -0700 Subject: [PATCH 1/4] output sweeps file should contain bitmask BRIGHTBLOB not logical BRIGHTSTARINBLOB --- bin/generate-sweep-files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/generate-sweep-files.py b/bin/generate-sweep-files.py index 9d1fe506d..f731c7ac1 100644 --- a/bin/generate-sweep-files.py +++ b/bin/generate-sweep-files.py @@ -442,7 +442,7 @@ def read_region(brickname, filename, bricksdesc): ('PMRA_IVAR', '>f4'), ('PMDEC', '>f4'), ('PMDEC_IVAR', '>f4'), - ('BRIGHTSTARINBLOB', '>b1')] + ('BRIGHTBLOB', '>i2')] ) def parse_args(): From 25572e3ec5be8450e8d013bfbefb667bbcebf24a Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Tue, 12 Mar 2019 13:57:56 -0700 Subject: [PATCH 2/4] assert that the data types are the same for columns in the sweeps and Tractor files --- bin/generate-sweep-files.py | 43 ++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/bin/generate-sweep-files.py b/bin/generate-sweep-files.py index f731c7ac1..b15be11e4 100644 --- a/bin/generate-sweep-files.py +++ b/bin/generate-sweep-files.py @@ -185,6 +185,19 @@ def filter(brickname, filename, region): return None, None else: raise + # ADM check all the column dtypes match. + if not ns.ignore_errors: + sflds = SWEEP_DTYPE.fields + tflds = objects.dtype.fields + for fld in sflds: + try: + sdt, tdt = sflds[fld][0], tflds[fld][0] + assert sdt == tdt + except: + msg = 'sweeps/Tractor dtypes differ for field ' + msg += '{}. Sweeps: {}, Tractor: {}'.format(fld, sdt, tdt) + raise ValueError(msg) + mask = objects['BRICK_PRIMARY'] != 0 objects = objects[mask] mask = objects['RA'] >= ra1 @@ -264,7 +277,7 @@ def read_region(brickname, filename, bricksdesc): SWEEP_DTYPE = np.dtype([ # ('BRICK_PRIMARY', '?'), - ('RELEASE', '>i4'), + ('RELEASE', '>i2'), ('BRICKID', '>i4'), ('BRICKNAME', 'S8'), ('OBJID', '>i4'), @@ -297,10 +310,10 @@ def read_region(brickname, filename, bricksdesc): ('FLUX_IVAR_I', '>f4'), ('FLUX_IVAR_Z', '>f4'), ('FLUX_IVAR_Y', '>f4'), - ('FLUX_IVAR_W1', '>f4'), - ('FLUX_IVAR_W2', '>f4'), - ('FLUX_IVAR_W3', '>f4'), - ('FLUX_IVAR_W4', '>f4'), + ('FLUX_IVAR_W1', '>f8'), + ('FLUX_IVAR_W2', '>f8'), + ('FLUX_IVAR_W3', '>f8'), + ('FLUX_IVAR_W4', '>f8'), # ('DECAM_MW_TRANSMISSION', '>f4', (6,)), # ('WISE_MW_TRANSMISSION', '>f4', (4,)), ('MW_TRANSMISSION_U', '>f4'), @@ -315,16 +328,16 @@ def read_region(brickname, filename, bricksdesc): ('MW_TRANSMISSION_W4', '>f4'), # ('DECAM_NOBS', 'u1', (6,)), # ('WISE_NOBS', '>i2', (4,)), - ('NOBS_U', 'i2'), - ('NOBS_G', 'i2'), - ('NOBS_R', 'i2'), - ('NOBS_I', 'i2'), - ('NOBS_Z', 'i2'), - ('NOBS_Y', 'i2'), - ('NOBS_W1', 'i2'), - ('NOBS_W2', 'i2'), - ('NOBS_W3', 'i2'), - ('NOBS_W4', 'i2'), + ('NOBS_U', '>i2'), + ('NOBS_G', '>i2'), + ('NOBS_R', '>i2'), + ('NOBS_I', '>i2'), + ('NOBS_Z', '>i2'), + ('NOBS_Y', '>i2'), + ('NOBS_W1', '>i2'), + ('NOBS_W2', '>i2'), + ('NOBS_W3', '>i2'), + ('NOBS_W4', '>i2'), # ('DECAM_RCHI2', '>f4', (6,)), # ('WISE_RCHI2', '>f4', (4,)), ('RCHISQ_U', '>f4'), From 126c8763de04ce2840423696435ac4cd7fa2aa02 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Tue, 12 Mar 2019 14:03:26 -0700 Subject: [PATCH 3/4] minor codes style clean-up --- bin/generate-sweep-files.py | 288 ++++++++++++++++++------------------ 1 file changed, 144 insertions(+), 144 deletions(-) diff --git a/bin/generate-sweep-files.py b/bin/generate-sweep-files.py index b15be11e4..482500378 100644 --- a/bin/generate-sweep-files.py +++ b/bin/generate-sweep-files.py @@ -277,156 +277,156 @@ def read_region(brickname, filename, bricksdesc): SWEEP_DTYPE = np.dtype([ # ('BRICK_PRIMARY', '?'), - ('RELEASE', '>i2'), - ('BRICKID', '>i4'), - ('BRICKNAME', 'S8'), - ('OBJID', '>i4'), - ('TYPE', 'S4'), - ('RA', '>f8'), - ('DEC', '>f8'), - ('RA_IVAR', '>f4'), + ('RELEASE', '>i2'), + ('BRICKID', '>i4'), + ('BRICKNAME', 'S8'), + ('OBJID', '>i4'), + ('TYPE', 'S4'), + ('RA', '>f8'), + ('DEC', '>f8'), + ('RA_IVAR', '>f4'), ('DEC_IVAR', '>f4'), - ('DCHISQ', '>f4', (5,)), + ('DCHISQ', '>f4', (5,)), ('EBV', '>f4'), -# ('TYCHO2INBLOB', '?'), -# ('OUT_OF_BOUNDS', '?'), -# ('DECAM_FLUX', '>f4', (6,)), -# ('WISE_FLUX', '>f4', (4,)), - ('FLUX_U', '>f4'), - ('FLUX_G', '>f4'), - ('FLUX_R', '>f4'), - ('FLUX_I', '>f4'), - ('FLUX_Z', '>f4'), - ('FLUX_Y', '>f4'), - ('FLUX_W1', '>f4'), - ('FLUX_W2', '>f4'), - ('FLUX_W3', '>f4'), - ('FLUX_W4', '>f4'), -# ('DECAM_FLUX_IVAR', '>f4', (6,)), -# ('WISE_FLUX_IVAR', '>f4', (4,)), - ('FLUX_IVAR_U', '>f4'), - ('FLUX_IVAR_G', '>f4'), - ('FLUX_IVAR_R', '>f4'), - ('FLUX_IVAR_I', '>f4'), - ('FLUX_IVAR_Z', '>f4'), - ('FLUX_IVAR_Y', '>f4'), - ('FLUX_IVAR_W1', '>f8'), - ('FLUX_IVAR_W2', '>f8'), - ('FLUX_IVAR_W3', '>f8'), - ('FLUX_IVAR_W4', '>f8'), -# ('DECAM_MW_TRANSMISSION', '>f4', (6,)), -# ('WISE_MW_TRANSMISSION', '>f4', (4,)), - ('MW_TRANSMISSION_U', '>f4'), - ('MW_TRANSMISSION_G', '>f4'), - ('MW_TRANSMISSION_R', '>f4'), - ('MW_TRANSMISSION_I', '>f4'), - ('MW_TRANSMISSION_Z', '>f4'), - ('MW_TRANSMISSION_Y', '>f4'), - ('MW_TRANSMISSION_W1', '>f4'), - ('MW_TRANSMISSION_W2', '>f4'), - ('MW_TRANSMISSION_W3', '>f4'), - ('MW_TRANSMISSION_W4', '>f4'), +# ('TYCHO2INBLOB', '?'), +# ('OUT_OF_BOUNDS', '?'), +# ('DECAM_FLUX', '>f4', (6,)), +# ('WISE_FLUX', '>f4', (4,)), + ('FLUX_U', '>f4'), + ('FLUX_G', '>f4'), + ('FLUX_R', '>f4'), + ('FLUX_I', '>f4'), + ('FLUX_Z', '>f4'), + ('FLUX_Y', '>f4'), + ('FLUX_W1', '>f4'), + ('FLUX_W2', '>f4'), + ('FLUX_W3', '>f4'), + ('FLUX_W4', '>f4'), +# ('DECAM_FLUX_IVAR', '>f4', (6,)), +# ('WISE_FLUX_IVAR', '>f4', (4,)), + ('FLUX_IVAR_U', '>f4'), + ('FLUX_IVAR_G', '>f4'), + ('FLUX_IVAR_R', '>f4'), + ('FLUX_IVAR_I', '>f4'), + ('FLUX_IVAR_Z', '>f4'), + ('FLUX_IVAR_Y', '>f4'), + ('FLUX_IVAR_W1', '>f8'), + ('FLUX_IVAR_W2', '>f8'), + ('FLUX_IVAR_W3', '>f8'), + ('FLUX_IVAR_W4', '>f8'), +# ('DECAM_MW_TRANSMISSION', '>f4', (6,)), +# ('WISE_MW_TRANSMISSION', '>f4', (4,)), + ('MW_TRANSMISSION_U', '>f4'), + ('MW_TRANSMISSION_G', '>f4'), + ('MW_TRANSMISSION_R', '>f4'), + ('MW_TRANSMISSION_I', '>f4'), + ('MW_TRANSMISSION_Z', '>f4'), + ('MW_TRANSMISSION_Y', '>f4'), + ('MW_TRANSMISSION_W1', '>f4'), + ('MW_TRANSMISSION_W2', '>f4'), + ('MW_TRANSMISSION_W3', '>f4'), + ('MW_TRANSMISSION_W4', '>f4'), # ('DECAM_NOBS', 'u1', (6,)), # ('WISE_NOBS', '>i2', (4,)), - ('NOBS_U', '>i2'), - ('NOBS_G', '>i2'), - ('NOBS_R', '>i2'), - ('NOBS_I', '>i2'), - ('NOBS_Z', '>i2'), - ('NOBS_Y', '>i2'), - ('NOBS_W1', '>i2'), - ('NOBS_W2', '>i2'), - ('NOBS_W3', '>i2'), - ('NOBS_W4', '>i2'), -# ('DECAM_RCHI2', '>f4', (6,)), -# ('WISE_RCHI2', '>f4', (4,)), - ('RCHISQ_U', '>f4'), - ('RCHISQ_G', '>f4'), - ('RCHISQ_R', '>f4'), - ('RCHISQ_I', '>f4'), - ('RCHISQ_Z', '>f4'), - ('RCHISQ_Y', '>f4'), - ('RCHISQ_W1', '>f4'), - ('RCHISQ_W2', '>f4'), - ('RCHISQ_W3', '>f4'), - ('RCHISQ_W4', '>f4'), -# ('DECAM_FRACFLUX', '>f4', (6,)), -# ('WISE_FRACFLUX', '>f4', (4,)), - ('FRACFLUX_U', '>f4'), - ('FRACFLUX_G', '>f4'), - ('FRACFLUX_R', '>f4'), - ('FRACFLUX_I', '>f4'), - ('FRACFLUX_Z', '>f4'), - ('FRACFLUX_Y', '>f4'), - ('FRACFLUX_W1', '>f4'), - ('FRACFLUX_W2', '>f4'), - ('FRACFLUX_W3', '>f4'), - ('FRACFLUX_W4', '>f4'), -# ('DECAM_FRACMASKED', '>f4', (6,)), - ('FRACMASKED_U', '>f4'), - ('FRACMASKED_G', '>f4'), - ('FRACMASKED_R', '>f4'), - ('FRACMASKED_I', '>f4'), - ('FRACMASKED_Z', '>f4'), - ('FRACMASKED_Y', '>f4'), + ('NOBS_U', '>i2'), + ('NOBS_G', '>i2'), + ('NOBS_R', '>i2'), + ('NOBS_I', '>i2'), + ('NOBS_Z', '>i2'), + ('NOBS_Y', '>i2'), + ('NOBS_W1', '>i2'), + ('NOBS_W2', '>i2'), + ('NOBS_W3', '>i2'), + ('NOBS_W4', '>i2'), +# ('DECAM_RCHI2', '>f4', (6,)), +# ('WISE_RCHI2', '>f4', (4,)), + ('RCHISQ_U', '>f4'), + ('RCHISQ_G', '>f4'), + ('RCHISQ_R', '>f4'), + ('RCHISQ_I', '>f4'), + ('RCHISQ_Z', '>f4'), + ('RCHISQ_Y', '>f4'), + ('RCHISQ_W1', '>f4'), + ('RCHISQ_W2', '>f4'), + ('RCHISQ_W3', '>f4'), + ('RCHISQ_W4', '>f4'), +# ('DECAM_FRACFLUX', '>f4', (6,)), +# ('WISE_FRACFLUX', '>f4', (4,)), + ('FRACFLUX_U', '>f4'), + ('FRACFLUX_G', '>f4'), + ('FRACFLUX_R', '>f4'), + ('FRACFLUX_I', '>f4'), + ('FRACFLUX_Z', '>f4'), + ('FRACFLUX_Y', '>f4'), + ('FRACFLUX_W1', '>f4'), + ('FRACFLUX_W2', '>f4'), + ('FRACFLUX_W3', '>f4'), + ('FRACFLUX_W4', '>f4'), +# ('DECAM_FRACMASKED', '>f4', (6,)), + ('FRACMASKED_U', '>f4'), + ('FRACMASKED_G', '>f4'), + ('FRACMASKED_R', '>f4'), + ('FRACMASKED_I', '>f4'), + ('FRACMASKED_Z', '>f4'), + ('FRACMASKED_Y', '>f4'), # ('DECAM_FRACIN', '>f4', (6,)), - ('FRACIN_U', '>f4'), - ('FRACIN_G', '>f4'), - ('FRACIN_R', '>f4'), - ('FRACIN_I', '>f4'), - ('FRACIN_Z', '>f4'), - ('FRACIN_Y', '>f4'), -# ('DECAM_ANYMASK', '>i2', (6,)), - ('ANYMASK_U', '>i2'), - ('ANYMASK_G', '>i2'), - ('ANYMASK_R', '>i2'), - ('ANYMASK_I', '>i2'), - ('ANYMASK_Z', '>i2'), - ('ANYMASK_Y', '>i2'), -# ('DECAM_ALLMASK', '>i2', (6,)), - ('ALLMASK_U', '>i2'), - ('ALLMASK_G', '>i2'), - ('ALLMASK_R', '>i2'), - ('ALLMASK_I', '>i2'), - ('ALLMASK_Z', '>i2'), - ('ALLMASK_Y', '>i2'), - ('WISEMASK_W1', 'u1'), + ('FRACIN_U', '>f4'), + ('FRACIN_G', '>f4'), + ('FRACIN_R', '>f4'), + ('FRACIN_I', '>f4'), + ('FRACIN_Z', '>f4'), + ('FRACIN_Y', '>f4'), +# ('DECAM_ANYMASK', '>i2', (6,)), + ('ANYMASK_U', '>i2'), + ('ANYMASK_G', '>i2'), + ('ANYMASK_R', '>i2'), + ('ANYMASK_I', '>i2'), + ('ANYMASK_Z', '>i2'), + ('ANYMASK_Y', '>i2'), +# ('DECAM_ALLMASK', '>i2', (6,)), + ('ALLMASK_U', '>i2'), + ('ALLMASK_G', '>i2'), + ('ALLMASK_R', '>i2'), + ('ALLMASK_I', '>i2'), + ('ALLMASK_Z', '>i2'), + ('ALLMASK_Y', '>i2'), + ('WISEMASK_W1', 'u1'), ('WISEMASK_W2', 'u1'), -# ('DECAM_PSFSIZE', '>f4', (6,)), - ('PSFSIZE_U', '>f4'), - ('PSFSIZE_G', '>f4'), - ('PSFSIZE_R', '>f4'), - ('PSFSIZE_I', '>f4'), - ('PSFSIZE_Z', '>f4'), - ('PSFSIZE_Y', '>f4'), -# ('DECAM_DEPTH', '>f4', (6,)), - ('PSFDEPTH_U', '>f4'), - ('PSFDEPTH_G', '>f4'), - ('PSFDEPTH_R', '>f4'), - ('PSFDEPTH_I', '>f4'), - ('PSFDEPTH_Z', '>f4'), - ('PSFDEPTH_Y', '>f4'), -# ('DECAM_GALDEPTH', '>f4', (6,)), - ('GALDEPTH_U', '>f4'), - ('GALDEPTH_G', '>f4'), - ('GALDEPTH_R', '>f4'), - ('GALDEPTH_I', '>f4'), - ('GALDEPTH_Z', '>f4'), - ('GALDEPTH_Y', '>f4'), - ('WISE_COADD_ID', 'S8'), - ('FRACDEV', '>f4'), - ('FRACDEV_IVAR', '>f4'), - ('SHAPEDEV_R', '>f4'), - ('SHAPEDEV_R_IVAR', '>f4'), - ('SHAPEDEV_E1', '>f4'), - ('SHAPEDEV_E1_IVAR', '>f4'), - ('SHAPEDEV_E2', '>f4'), - ('SHAPEDEV_E2_IVAR', '>f4'), - ('SHAPEEXP_R', '>f4'), - ('SHAPEEXP_R_IVAR', '>f4'), - ('SHAPEEXP_E1', '>f4'), - ('SHAPEEXP_E1_IVAR', '>f4'), - ('SHAPEEXP_E2', '>f4'), +# ('DECAM_PSFSIZE', '>f4', (6,)), + ('PSFSIZE_U', '>f4'), + ('PSFSIZE_G', '>f4'), + ('PSFSIZE_R', '>f4'), + ('PSFSIZE_I', '>f4'), + ('PSFSIZE_Z', '>f4'), + ('PSFSIZE_Y', '>f4'), +# ('DECAM_DEPTH', '>f4', (6,)), + ('PSFDEPTH_U', '>f4'), + ('PSFDEPTH_G', '>f4'), + ('PSFDEPTH_R', '>f4'), + ('PSFDEPTH_I', '>f4'), + ('PSFDEPTH_Z', '>f4'), + ('PSFDEPTH_Y', '>f4'), +# ('DECAM_GALDEPTH', '>f4', (6,)), + ('GALDEPTH_U', '>f4'), + ('GALDEPTH_G', '>f4'), + ('GALDEPTH_R', '>f4'), + ('GALDEPTH_I', '>f4'), + ('GALDEPTH_Z', '>f4'), + ('GALDEPTH_Y', '>f4'), + ('WISE_COADD_ID', 'S8'), + ('FRACDEV', '>f4'), + ('FRACDEV_IVAR', '>f4'), + ('SHAPEDEV_R', '>f4'), + ('SHAPEDEV_R_IVAR', '>f4'), + ('SHAPEDEV_E1', '>f4'), + ('SHAPEDEV_E1_IVAR', '>f4'), + ('SHAPEDEV_E2', '>f4'), + ('SHAPEDEV_E2_IVAR', '>f4'), + ('SHAPEEXP_R', '>f4'), + ('SHAPEEXP_R_IVAR', '>f4'), + ('SHAPEEXP_E1', '>f4'), + ('SHAPEEXP_E1_IVAR', '>f4'), + ('SHAPEEXP_E2', '>f4'), ('SHAPEEXP_E2_IVAR', '>f4'), ('FIBERFLUX_U', '>f4'), ('FIBERFLUX_G', '>f4'), From 50a7cd474402eba161a10095bfc88cff732e5d86 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Tue, 12 Mar 2019 14:17:03 -0700 Subject: [PATCH 4/4] code style and quality for codacy checks --- bin/generate-sweep-files.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/bin/generate-sweep-files.py b/bin/generate-sweep-files.py index 482500378..ae81e5849 100644 --- a/bin/generate-sweep-files.py +++ b/bin/generate-sweep-files.py @@ -190,10 +190,8 @@ def filter(brickname, filename, region): sflds = SWEEP_DTYPE.fields tflds = objects.dtype.fields for fld in sflds: - try: - sdt, tdt = sflds[fld][0], tflds[fld][0] - assert sdt == tdt - except: + sdt, tdt = sflds[fld][0], tflds[fld][0] + if sdt != tdt: msg = 'sweeps/Tractor dtypes differ for field ' msg += '{}. Sweeps: {}, Tractor: {}'.format(fld, sdt, tdt) raise ValueError(msg) @@ -327,7 +325,7 @@ def read_region(brickname, filename, bricksdesc): ('MW_TRANSMISSION_W3', '>f4'), ('MW_TRANSMISSION_W4', '>f4'), # ('DECAM_NOBS', 'u1', (6,)), -# ('WISE_NOBS', '>i2', (4,)), +# ('WISE_NOBS', '>i2', (4,)), ('NOBS_U', '>i2'), ('NOBS_G', '>i2'), ('NOBS_R', '>i2'), @@ -460,7 +458,7 @@ def read_region(brickname, filename, bricksdesc): def parse_args(): ap = argparse.ArgumentParser( - description="""Create Sweep files for DECALS. + description="""Create Sweep files for DECALS. This tool ensures each sweep file contains roughly '-n' objects. HDF5 and FITS formats are supported. Columns contained in a sweep file are: @@ -479,23 +477,23 @@ def parse_args(): ap.add_argument('-F', "--filelist", default=None, help="list of tractor brickfiles to use; this will avoid expensive walking of the path.") - ap.add_argument('-d', "--bricksdesc", default=None, + ap.add_argument('-d', "--bricksdesc", default=None, help="location of decals-bricks.fits, speeds up the scanning") ap.add_argument('-v', "--verbose", action='store_true') ap.add_argument('-I', "--ignore-errors", action='store_true') - ap.add_argument('-S', "--schema", choices=['blocks', 'dec', 'ra'], - default='blocks', + ap.add_argument('-S', "--schema", choices=['blocks', 'dec', 'ra'], + default='blocks', help="""Decomposition schema. Still being tuned. """) - ap.add_argument('-b', "--bricklist", - help="""Filename with list of bricknames to include. + ap.add_argument('-b', "--bricklist", + help="""Filename with list of bricknames to include. If not set, all bricks in src are included, sorted by brickname. """) ap.add_argument("--numproc", type=int, default=None, - help="""Number of concurrent processes to use. 0 for sequential execution. + help="""Number of concurrent processes to use. 0 for sequential execution. Default is to use OMP_NUM_THREADS, or the number of cores on the node.""") return ap.parse_args()