allowing custom output path, fixed parbin bug

settings/default_inputs.yml

@@ -4,11 +4,14 @@
 
 local:
   # shear catalogs in xshear style
-  shearcat:
-    default: default.dat
+  shearcat: 
+    default_shear: #shear_to_use
+      shuf_y1a1-unblind-v2-mcal_zmean-meta_zmc-mof.dat
 
   # redmapper catalogs
   lenscat:
-    default:
-      lens: default.dat
-      rand: default.dat
+    default_cat:
+      lens: 
+        y1a1_gold_1.0.3-d10-mof-001d_run_redmapper_v6.4.17-vlim_lgt5_desformat_catalog.fit
+      rand: 
+        y1a1_gold_1.0.3_wide+d10-mof-001d_run_redmapper_v6.4.17-2_randcat_z0.10-0.95_lgt005_vl02.fit

settings/default_params.yml

@@ -19,11 +19,16 @@
 
 # If you want to use an other parameter file, then
 # specify it here. It must be in the same directory
-custom_params_file: params.yml
+custom_params_file: iotest_params.yml
 
 # if False: uses default project_path + /data
-custom_data_path: False
-pdf_paths: null
+custom_data_path: /global/cscratch1/sd/tvarga/Y1_DATA/xpipe_data/
+pdf_paths: /global/cscratch1/sd/tvarga/Y1_DATA/DES_PDF/*.h5
+
+# Heidi
+custom_output_path: False #/global/cscratch1/sd/haoyiwu/xpipe_output/iotest/
+
+
 
 ######################################################
 
@@ -35,14 +40,15 @@ pdf_paths: null
 mode: full
 
 # prefix for all files (with NO trailing "_")
-tag: default
+tag: test #desy1run
 
 # reduced, lensfit, metacal
-shear_style: reduced
+shear_style: metacal
 
 # aliases for catalogs defined in default_inputs.yml
-shear_to_use: default
-cat_to_use: default
+## Heidi: these two cannot be the same
+shear_to_use: default_shear  # corresponds to shearcat in default_inputs.yml
+cat_to_use: default_cat  # corresponds to lenscat in default_inputs.yml
 
 # parameter bins
 param_bins_full:
@@ -58,7 +64,7 @@ nprocess: 2
 
 # maximum number of Jackknife regions to use in resampling
 # actual number is max(n_lens, njk_max)
-njk_max: 100
+njk_max: 2 #100
 
 # number of random points to use
 nrandoms:
@@ -92,9 +98,10 @@ weight_style: "optimal"
 
 # source-lens pair logging
 pairlog:
- pairlog_rmin: 0
- pairlog_rmax: 0
- pairlog_nmax: 0
+ pairlog_rmin: 0 #r bin numbers between which pairs are printed to file; 
+ pairlog_rmax: 15 #
+ pairlog_nmax: 0 # maximum number of pairs printed in each radial bin; <=0 for no maximum
+
 
 
 ######################################################
@@ -121,6 +128,7 @@ randkey:
 
 lens_prefix: y1clust
 rand_prefix: y1rand
+subtr_prefix: y1subtr ## Heidi: required if using randoms for postprocess.py 
 
 suffixes:
   "profile": "_profile.dat"
@@ -168,11 +176,17 @@ pzpars:
   full:
     tag: "zpdf"
   boost:
-    rbmin: 3
-    rbmax: 13
+    refprof: 14
+    rbmin: 0.15 #3 # Heidi: not used?
+    rbmax: 0.25 #13# Heidi: not used?
 
+#### below added by Heidi
+source_bins_to_use: [0] # Tamas says it has no effect
 
 
+### below: used by shape noise
+shear_nrot: 1 # TODO: probably need bigger
 
+shear_seed_master: 0 
 
 

settings/default_paths.yml

@@ -1,11 +1,18 @@
 ###################################
 #   DATA directory tree
 ###################################
-
 data:
-  shearcat: /shearcat
+  shearcat: /  #/shearcat
   lenscat: /lenscat
+
+
+
+
+###################################
+#   output directory tree  # Heidi Sep 2019
+###################################
+output:
   xin: /xshear_in
   xout: /xshear_out
   results: /results
-  calibs: /calibs
+  calibs: /calibs

settings/iotest_params.yml

@@ -0,0 +1,194 @@
+######################################################
+#  Default Parameter bins for Y1 analysis
+#
+#  The values defined here are loaded into
+#  xpipe.paths.params
+#
+#  LOAD ORDER:
+#  -------------------------
+#  1) default_params.yml
+#  2) looks for params.yml
+#  3) tries to read "custom_params_file" from params.yml
+#
+#  from this point the load is recursive, e.g. param
+#  files are loaded as long as there is a valid
+#  custom params file defined in the last loaded
+#  config
+#
+######################################################
+
+# If you want to use an other parameter file, then
+# specify it here. It must be in the same directory
+custom_params_file: iotest_params.yml
+
+# if False: uses default project_path + /data
+custom_data_path: /global/cscratch1/sd/tvarga/Y1_DATA/xpipe_data/
+pdf_paths: /global/cscratch1/sd/tvarga/Y1_DATA/DES_PDF/*.h5
+
+## added by Heidi 
+# if False: uses default 
+custom_output_path: /global/cscratch1/sd/haoyiwu/xpipe_output/iotest/
+
+
+
+
+######################################################
+
+# The pipeline supports two modes: full and dev
+# This is primarily used in setting up the input files
+# for the measurement. E.g. you can define two binning
+# schemes: one really complex for full, and a simple
+# which is expected to run much faster for dev
+mode: dev
+
+# prefix for all files (with NO trailing "_")
+tag: iotest #desy1run
+
+# reduced, lensfit, metacal
+shear_style: metacal
+
+# aliases for catalogs defined in default_inputs.yml
+## Heidi: these two cannot be the same
+shear_to_use: default_shear  # corresponds to shearcat in default_inputs.yml
+cat_to_use: default_cat  # corresponds to lenscat in default_inputs.yml
+
+# parameter bins
+param_bins_full:
+  q0_edges: [0.2, 0.35, 0.5, 0.65]
+  q1_edges: [5., 10., 14., 20., 30., 45., 60., 999]
+
+param_bins_dev:
+  q0_edges: [0.2, 0.35]
+  q1_edges: [45, 60]
+
+# number of processes (cores) to use in calculations
+nprocess: 2
+
+# maximum number of Jackknife regions to use in resampling
+# actual number is max(n_lens, njk_max)
+njk_max: 2 #100
+
+# number of random points to use
+nrandoms:
+  full: 50000
+  dev: 1000
+
+# random seed for choosing the random points
+seeds:
+  random_seed: 5
+  shear_seed_master: 10
+
+######################################################
+# Here define parameters for XSHEAR config
+
+# cosmology parameters:
+cosmo_params:
+  H0: 70.
+  Om0: 0.3
+
+# logarithmic (base 10) radial bins from rmin to rmax
+# units: Mpc, comoving_mpc or arcmin
+radial_bins:
+  nbin: 15
+  rmin: 0.0323
+  rmax: 30.0
+  units: Mpc
+
+# source weight style
+# "optimal" for DeltaSigma, or "uniform" for shear
+weight_style: "optimal"
+
+# source-lens pair logging
+pairlog:
+ pairlog_rmin: 0 #r bin numbers between which pairs are printed to file; 
+ pairlog_rmax: 15 #
+ pairlog_nmax: 0 # maximum number of pairs printed in each radial bin; <=0 for no maximum
+
+
+
+######################################################
+# aliases for catolog parameters
+
+lenskey:
+  id: MEM_MATCH_ID
+  ra: RA
+  dec: DEC
+  z: Z_LAMBDA
+  q0: Z_LAMBDA
+  q1: LAMBDA_CHISQ
+
+randkey:
+  q0: ZTRUE
+  q1: AVG_LAMBDAOUT
+  ra: RA
+  dec: DEC
+  z: ZTRUE
+  w: WEIGHT
+
+######################################################
+# file prefixes and suffixes
+
+lens_prefix: y1clust
+rand_prefix: y1rand
+subtr_prefix: y1subtr ## Heidi: required if using randoms for postprocess.py 
+
+suffixes:
+  "profile": "_profile.dat"
+  "dst_cov": "_dst_cov.dat"
+  "dsx_cov": "_dsx_cov.dat"
+  "radial_snum": "_radial_snum"
+  "corr_boost": "_corr_boost.dat"
+  "corr_boost_cov": "_corr_boost_cov.dat"
+  "calibs_log": "_calibs_log.dat"
+
+
+
+######################################################
+# DES fields with rectangular boundary in radec
+
+# as defined below
+fields_to_use: ['spt', 's82']
+
+fields:
+  spt:
+    dec_top: -30.
+    dec_bottom: -60.
+    ra_left: 0.
+    ra_right: 360.
+  s82:
+    dec_top: 10.
+    dec_bottom: -10.
+    ra_left: 300.
+    ra_right: 10.
+  d04:
+    dec_top: 10.
+    dec_bottom: -30.
+    ra_left: 10.
+    ra_right: 250.
+
+######################################################
+# boost factor p(z) params
+
+pzpars:
+  hist:
+    nbin: 15
+    zmin: 0.0
+    zmax: 3.0
+    tag: "zhist"
+  full:
+    tag: "zpdf"
+  boost:
+    refprof: 14
+    rbmin: 0.15 #3 # Heidi: not used?
+    rbmax: 0.25 #13# Heidi: not used?
+
+#### below added by Heidi
+source_bins_to_use: [0] # Tamas says it has no effect
+
+
+### below: used by shape noise
+shear_nrot: 1 # TODO: probably need bigger
+
+shear_seed_master: 0 
+
+

xpipe/paths.py

@@ -84,6 +84,16 @@ def get_dirpaths(params, project_path):
     if not custom_data_path:
         custom_data_path = project_path + 'data'
     _dirpaths = expand_paths(_dirs['data'], custom_data_path)
+
+
+    ## Heidi
+    custom_output_path = params['custom_output_path']
+    if not custom_output_path:
+        custom_output_path = custom_data_path
+    else:
+        print('custom output path', custom_output_path)
+    _dirpaths.update(expand_paths(_dirs['output'], custom_output_path))
+
     return _dirpaths
 
 
@@ -231,12 +241,13 @@ def get_bin_settings(params, devmode):
 
     # READING custom params files
     has_custom_specified = "custom_params_file" in params.keys()
-    while has_custom_specified and params["custom_params_file"] is not None:
+    #while has_custom_specified and params["custom_params_file"] is not None:#Heidi
+    if has_custom_specified and params["custom_params_file"] is not None:
         custom_param_path = _complete_params_path(params["custom_params_file"])
-        if os.path.isfile(custom_param_path):
-            _update_params(custom_param_path)
-        else:
-            break
+        #if os.path.isfile(custom_param_path): # Heidi: uncomment, infinite loop
+        _update_params(custom_param_path)
+        #else:
+        #    break
 
 
     # TODO add effective re-initialization of params for API mode
@@ -245,3 +256,4 @@ def get_bin_settings(params, devmode):
     def set_params(**kwargs):
         global devmode, dirpaths, fullpaths, fullurls, pdf_files
 
+