From 8d1c786bd9644843a2cd15a47b0a98f8627e8a70 Mon Sep 17 00:00:00 2001 From: Heidi Date: Fri, 27 Sep 2019 05:38:20 -0700 Subject: [PATCH] allowing custom output path, fixed parbin bug --- settings/default_inputs.yml | 13 ++- settings/default_params.yml | 40 +++++--- settings/default_paths.yml | 13 ++- settings/iotest_params.yml | 194 ++++++++++++++++++++++++++++++++++++ xpipe/paths.py | 22 +++- xpipe/xhandle/parbins.py | 15 +-- 6 files changed, 265 insertions(+), 32 deletions(-) create mode 100644 settings/iotest_params.yml diff --git a/settings/default_inputs.yml b/settings/default_inputs.yml index cf6f45b..27cfb82 100644 --- a/settings/default_inputs.yml +++ b/settings/default_inputs.yml @@ -4,11 +4,14 @@ local: # shear catalogs in xshear style - shearcat: - default: default.dat + shearcat: + default_shear: #shear_to_use + shuf_y1a1-unblind-v2-mcal_zmean-meta_zmc-mof.dat # redmapper catalogs lenscat: - default: - lens: default.dat - rand: default.dat + default_cat: + lens: + y1a1_gold_1.0.3-d10-mof-001d_run_redmapper_v6.4.17-vlim_lgt5_desformat_catalog.fit + rand: + y1a1_gold_1.0.3_wide+d10-mof-001d_run_redmapper_v6.4.17-2_randcat_z0.10-0.95_lgt005_vl02.fit diff --git a/settings/default_params.yml b/settings/default_params.yml index 1c36e57..9e41bf6 100644 --- a/settings/default_params.yml +++ b/settings/default_params.yml @@ -19,11 +19,16 @@ # If you want to use an other parameter file, then # specify it here. It must be in the same directory -custom_params_file: params.yml +custom_params_file: iotest_params.yml # if False: uses default project_path + /data -custom_data_path: False -pdf_paths: null +custom_data_path: /global/cscratch1/sd/tvarga/Y1_DATA/xpipe_data/ +pdf_paths: /global/cscratch1/sd/tvarga/Y1_DATA/DES_PDF/*.h5 + +# Heidi +custom_output_path: False #/global/cscratch1/sd/haoyiwu/xpipe_output/iotest/ + + ###################################################### @@ -35,14 +40,15 @@ pdf_paths: null mode: full # prefix for all files (with NO trailing "_") -tag: default +tag: test #desy1run # reduced, lensfit, metacal -shear_style: reduced +shear_style: metacal # aliases for catalogs defined in default_inputs.yml -shear_to_use: default -cat_to_use: default +## Heidi: these two cannot be the same +shear_to_use: default_shear # corresponds to shearcat in default_inputs.yml +cat_to_use: default_cat # corresponds to lenscat in default_inputs.yml # parameter bins param_bins_full: @@ -58,7 +64,7 @@ nprocess: 2 # maximum number of Jackknife regions to use in resampling # actual number is max(n_lens, njk_max) -njk_max: 100 +njk_max: 2 #100 # number of random points to use nrandoms: @@ -92,9 +98,10 @@ weight_style: "optimal" # source-lens pair logging pairlog: - pairlog_rmin: 0 - pairlog_rmax: 0 - pairlog_nmax: 0 + pairlog_rmin: 0 #r bin numbers between which pairs are printed to file; + pairlog_rmax: 15 # + pairlog_nmax: 0 # maximum number of pairs printed in each radial bin; <=0 for no maximum + ###################################################### @@ -121,6 +128,7 @@ randkey: lens_prefix: y1clust rand_prefix: y1rand +subtr_prefix: y1subtr ## Heidi: required if using randoms for postprocess.py suffixes: "profile": "_profile.dat" @@ -168,11 +176,17 @@ pzpars: full: tag: "zpdf" boost: - rbmin: 3 - rbmax: 13 + refprof: 14 + rbmin: 0.15 #3 # Heidi: not used? + rbmax: 0.25 #13# Heidi: not used? +#### below added by Heidi +source_bins_to_use: [0] # Tamas says it has no effect +### below: used by shape noise +shear_nrot: 1 # TODO: probably need bigger +shear_seed_master: 0 diff --git a/settings/default_paths.yml b/settings/default_paths.yml index ae2f9d8..886e1b3 100644 --- a/settings/default_paths.yml +++ b/settings/default_paths.yml @@ -1,11 +1,18 @@ ################################### # DATA directory tree ################################### - data: - shearcat: /shearcat + shearcat: / #/shearcat lenscat: /lenscat + + + + +################################### +# output directory tree # Heidi Sep 2019 +################################### +output: xin: /xshear_in xout: /xshear_out results: /results - calibs: /calibs \ No newline at end of file + calibs: /calibs diff --git a/settings/iotest_params.yml b/settings/iotest_params.yml new file mode 100644 index 0000000..b9a3532 --- /dev/null +++ b/settings/iotest_params.yml @@ -0,0 +1,194 @@ +###################################################### +# Default Parameter bins for Y1 analysis +# +# The values defined here are loaded into +# xpipe.paths.params +# +# LOAD ORDER: +# ------------------------- +# 1) default_params.yml +# 2) looks for params.yml +# 3) tries to read "custom_params_file" from params.yml +# +# from this point the load is recursive, e.g. param +# files are loaded as long as there is a valid +# custom params file defined in the last loaded +# config +# +###################################################### + +# If you want to use an other parameter file, then +# specify it here. It must be in the same directory +custom_params_file: iotest_params.yml + +# if False: uses default project_path + /data +custom_data_path: /global/cscratch1/sd/tvarga/Y1_DATA/xpipe_data/ +pdf_paths: /global/cscratch1/sd/tvarga/Y1_DATA/DES_PDF/*.h5 + +## added by Heidi +# if False: uses default +custom_output_path: /global/cscratch1/sd/haoyiwu/xpipe_output/iotest/ + + + + +###################################################### + +# The pipeline supports two modes: full and dev +# This is primarily used in setting up the input files +# for the measurement. E.g. you can define two binning +# schemes: one really complex for full, and a simple +# which is expected to run much faster for dev +mode: dev + +# prefix for all files (with NO trailing "_") +tag: iotest #desy1run + +# reduced, lensfit, metacal +shear_style: metacal + +# aliases for catalogs defined in default_inputs.yml +## Heidi: these two cannot be the same +shear_to_use: default_shear # corresponds to shearcat in default_inputs.yml +cat_to_use: default_cat # corresponds to lenscat in default_inputs.yml + +# parameter bins +param_bins_full: + q0_edges: [0.2, 0.35, 0.5, 0.65] + q1_edges: [5., 10., 14., 20., 30., 45., 60., 999] + +param_bins_dev: + q0_edges: [0.2, 0.35] + q1_edges: [45, 60] + +# number of processes (cores) to use in calculations +nprocess: 2 + +# maximum number of Jackknife regions to use in resampling +# actual number is max(n_lens, njk_max) +njk_max: 2 #100 + +# number of random points to use +nrandoms: + full: 50000 + dev: 1000 + +# random seed for choosing the random points +seeds: + random_seed: 5 + shear_seed_master: 10 + +###################################################### +# Here define parameters for XSHEAR config + +# cosmology parameters: +cosmo_params: + H0: 70. + Om0: 0.3 + +# logarithmic (base 10) radial bins from rmin to rmax +# units: Mpc, comoving_mpc or arcmin +radial_bins: + nbin: 15 + rmin: 0.0323 + rmax: 30.0 + units: Mpc + +# source weight style +# "optimal" for DeltaSigma, or "uniform" for shear +weight_style: "optimal" + +# source-lens pair logging +pairlog: + pairlog_rmin: 0 #r bin numbers between which pairs are printed to file; + pairlog_rmax: 15 # + pairlog_nmax: 0 # maximum number of pairs printed in each radial bin; <=0 for no maximum + + + +###################################################### +# aliases for catolog parameters + +lenskey: + id: MEM_MATCH_ID + ra: RA + dec: DEC + z: Z_LAMBDA + q0: Z_LAMBDA + q1: LAMBDA_CHISQ + +randkey: + q0: ZTRUE + q1: AVG_LAMBDAOUT + ra: RA + dec: DEC + z: ZTRUE + w: WEIGHT + +###################################################### +# file prefixes and suffixes + +lens_prefix: y1clust +rand_prefix: y1rand +subtr_prefix: y1subtr ## Heidi: required if using randoms for postprocess.py + +suffixes: + "profile": "_profile.dat" + "dst_cov": "_dst_cov.dat" + "dsx_cov": "_dsx_cov.dat" + "radial_snum": "_radial_snum" + "corr_boost": "_corr_boost.dat" + "corr_boost_cov": "_corr_boost_cov.dat" + "calibs_log": "_calibs_log.dat" + + + +###################################################### +# DES fields with rectangular boundary in radec + +# as defined below +fields_to_use: ['spt', 's82'] + +fields: + spt: + dec_top: -30. + dec_bottom: -60. + ra_left: 0. + ra_right: 360. + s82: + dec_top: 10. + dec_bottom: -10. + ra_left: 300. + ra_right: 10. + d04: + dec_top: 10. + dec_bottom: -30. + ra_left: 10. + ra_right: 250. + +###################################################### +# boost factor p(z) params + +pzpars: + hist: + nbin: 15 + zmin: 0.0 + zmax: 3.0 + tag: "zhist" + full: + tag: "zpdf" + boost: + refprof: 14 + rbmin: 0.15 #3 # Heidi: not used? + rbmax: 0.25 #13# Heidi: not used? + +#### below added by Heidi +source_bins_to_use: [0] # Tamas says it has no effect + + +### below: used by shape noise +shear_nrot: 1 # TODO: probably need bigger + +shear_seed_master: 0 + + diff --git a/xpipe/paths.py b/xpipe/paths.py index 27f56f1..ce8e4fb 100644 --- a/xpipe/paths.py +++ b/xpipe/paths.py @@ -84,6 +84,16 @@ def get_dirpaths(params, project_path): if not custom_data_path: custom_data_path = project_path + 'data' _dirpaths = expand_paths(_dirs['data'], custom_data_path) + + + ## Heidi + custom_output_path = params['custom_output_path'] + if not custom_output_path: + custom_output_path = custom_data_path + else: + print('custom output path', custom_output_path) + _dirpaths.update(expand_paths(_dirs['output'], custom_output_path)) + return _dirpaths @@ -231,12 +241,13 @@ def get_bin_settings(params, devmode): # READING custom params files has_custom_specified = "custom_params_file" in params.keys() - while has_custom_specified and params["custom_params_file"] is not None: + #while has_custom_specified and params["custom_params_file"] is not None:#Heidi + if has_custom_specified and params["custom_params_file"] is not None: custom_param_path = _complete_params_path(params["custom_params_file"]) - if os.path.isfile(custom_param_path): - _update_params(custom_param_path) - else: - break + #if os.path.isfile(custom_param_path): # Heidi: uncomment, infinite loop + _update_params(custom_param_path) + #else: + # break # TODO add effective re-initialization of params for API mode @@ -245,3 +256,4 @@ def get_bin_settings(params, devmode): def set_params(**kwargs): global devmode, dirpaths, fullpaths, fullurls, pdf_files + diff --git a/xpipe/xhandle/parbins.py b/xpipe/xhandle/parbins.py index 199ede3..d2c2e38 100644 --- a/xpipe/xhandle/parbins.py +++ b/xpipe/xhandle/parbins.py @@ -356,7 +356,7 @@ def load_randcat(params=None, fullpaths=None, which=None): select = np.ones(len(ra), dtype=bool) # number of parameter columns - nq = len(randkey.keys()) - 3 + nq = len(randkey.keys()) - 4 # Heidi if "jkey" in randkey.keys(): nq -= 1 qlist = np.zeros(shape=(len(ra), nq)) @@ -736,15 +736,18 @@ def mkdir(self): """creates project directory""" self.dpath = self.dirpaths['xin'] + "/" + self.params["tag"] if not os.path.isdir(self.dpath): - os.mkdir(self.dpath) + #os.mkdir(self.dpath) + os.makedirs(self.dpath) # Heidi self.opath = self.dirpaths['xout'] + "/" + self.params["tag"] if not os.path.isdir(self.opath): - os.mkdir(self.opath) + #os.mkdir(self.opath) + os.makedirs(self.opath) # Heidi self.respath = self.dirpaths['results'] + "/" + self.params["tag"] if not os.path.isdir(self.respath): - os.mkdir(self.respath) + #os.mkdir(self.respath) + os.makedirs(self.respath) # Heidi def _save_jk_cens(self): fname = self.dpath + '/' + self.params["tag"] + "_jkcens" + self.bin_tag + '.dat' @@ -756,7 +759,7 @@ def save_clust(self): makecat(self.dpath + "/" + self.flist[self.ind], self.lenses['id'][sind], self.lenses['ra'][sind], self.lenses['dec'][sind], self.lenses['z'][sind]) - print('saved ' + self.flist[self.ind]) + print('saved ' + self.dpath + "/" +self.flist[self.ind])# Heidi fio.write(self.dpath + "/" + self.flist[self.ind].replace('.dat', '.fits'), self.lenses["fullcat"][sind], clobber=True) @@ -837,7 +840,7 @@ def save_rands(self): makecat(self.dpath + "/" + self.rlist[self.ind], self.randoms['id'][rind][self.idraw], self.randoms['ra'][rind][self.idraw], self.randoms['dec'][rind][self.idraw], self.randoms['z'][rind][self.idraw]) - print('saved ' + self.rlist[self.ind]) + print('saved ' + self.dpath + "/" +self.rlist[self.ind]) #Heidi def save_rands_jk(self): """writes random points to file for each JK patch in xshear style"""