Merge pull request #314 from LSSTDESC/issue_#311_create_CL_make_bin_s…

…tage

Issue #311 create cl make bin stage

examples/cosmodc2/1deg2-in2p3.sub

@@ -0,0 +1,9 @@
+#!/usr/bin/bash
+#SBATCH --time=01:00:00
+#SBATCH --partition=hpc
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=1
+#SBATCH --mem=128000
+
+source /pbs/throng/lsst/users/jzuntz/txpipe-environments/setup-txpipe
+ceci examples/cosmodc2/pipeline-1deg2-CL-in2p3.yml

examples/cosmodc2/1deg2-nersc.sub

@@ -0,0 +1,10 @@
+#!/bin/bash
+#SBATCH -A m1727
+#SBATCH -C cpu
+#SBATCH --qos=debug
+#SBATCH --time=00:30:00
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=32
+
+source $CFS/lsst/groups/WL/users/zuntz/setup-txpipe
+tx ceci examples/cosmodc2/pipeline-1deg2-CL-nersc.yml

examples/cosmodc2/20deg2-in2p3.sub

@@ -6,4 +6,4 @@
 #SBATCH --mem=128000
 
 source /pbs/throng/lsst/users/jzuntz/txpipe-environments/setup-txpipe
-ceci examples/cosmodc2/pipeline-20deg2-clmm.yml
+ceci examples/cosmodc2/pipeline-20deg2-CL-in2p3.yml

examples/cosmodc2/20deg2-nersc.sub

@@ -7,5 +7,4 @@
 #SBATCH --ntasks-per-node=32
 
 source $CFS/lsst/groups/WL/users/zuntz/setup-txpipe
-
-tx ceci examples/cosmodc2/pipeline-20deg2-clmm-nersc.yml
+tx ceci examples/cosmodc2/pipeline-20deg2-CL-nersc.yml

examples/cosmodc2/config-20deg2-clmm.yml → examples/cosmodc2/config-1deg2-CL.yml

@@ -63,7 +63,25 @@ BPZ_lite:
         mag_i: 28.62
         mag_z: 27.98
 
+CLClusterBinningRedshiftRichness:
+    zedge : [0.1, 0.4, 0.6, 0.8]
+    richedge : [5., 10., 20.,25.]
 
 CLClusterShearCatalogs:
     max_radius: 5.0  # Mpc
     delta_z: 0.2  # redshift buffer
+
+CLClusterEnsembleProfiles:
+    #radial bin definition
+    r_min : 0.2 #in Mpc
+    r_max : 5.0 #in Mpc
+    #type of profile
+    delta_sigma_profile : True
+    shear_profile : False
+    magnification_profile : False
+    #stacking step or not
+    stack_profile : True
+
+
+
+

examples/cosmodc2/config-20deg2-CL.yml

@@ -0,0 +1,88 @@
+TXSourceSelectorMetadetect:
+    input_pz: False
+    bands: riz #used for selection
+    T_cut: 0.5
+    s2n_cut: 10.0
+    max_rows: 1000
+    delta_gamma: 0.02
+    source_zbin_edges: [0.1, 3.0]
+    chunk_rows: 100000
+    true_z: False
+    shear_prefix: ''
+
+Inform_BPZ_lite:
+    aliases:
+        input: spectroscopic_catalog
+        model: photoz_model
+    zmin: 0.0
+    zmax: 3.0
+    nzbins: 301
+    columns_file: ./data/bpz_riz.columns
+    data_path: ./data/example/rail-bpz-inputs
+    spectra_file: CWWSB4.list
+    prior_band: i
+    ref_band: i
+    # Not sure about this
+    prior_file: hdfn_gen
+    p_min: 0.005
+    gauss_kernel: 0.0
+    mag_err_min: 0.005
+    inform_options: {'save_train': False, 'load_model': False, 'modelfile': 'BPZpriormodel.out'}  
+    madau_reddening: no
+    bands: riz
+    zp_errors: [0.01, 0.01, 0.01]
+    hdf5_groupname: photometry
+
+
+
+BPZ_lite:
+    aliases:
+        model: photoz_model
+        input: shear_catalog
+        output: source_photoz_pdfs
+    zmin: 0.0
+    zmax: 3.0
+    dz: 0.01
+    nzbins: 301
+    data_path: ./data/example/rail-bpz-inputs
+    bands: [mag_r, mag_i, mag_z]
+    err_bands: [mag_err_r, mag_err_i, mag_err_z]
+    hdf5_groupname: shear/00
+    nondetect_val: .inf
+    columns_file: ./data/bpz_riz.columns
+    spectra_file: CWWSB4.list
+    ref_band: mag_i
+    prior_file: hdfn_gen
+    p_min: 0.005
+    gauss_kernel: 0.0
+    zp_errors: [0.01, 0.01, 0.01]
+    mag_err_min: 0.005
+    madau_reddening: false
+    mag_limits:
+        mag_r: 29.06
+        mag_i: 28.62
+        mag_z: 27.98
+
+CLClusterBinningRedshiftRichness:
+    zedge : [0.1, 0.4, 0.6, 0.8]
+    richedge : [5., 10., 20.,25.]
+
+CLClusterShearCatalogs:
+    max_radius: 5.0  # Mpc
+    delta_z: 0.2  # redshift buffer
+
+CLClusterEnsembleProfiles:
+    #radial bin definition
+    r_min : 0.2 #in Mpc
+    r_max : 5.0 #in Mpc
+    #type of profile
+    delta_sigma_profile : True
+    shear_profile : False
+    magnification_profile : False
+    #stacking step or not
+    stack_profile : True
+
+
+
+
+

examples/cosmodc2/pipeline-1deg2-CL-in2p3.yml

@@ -0,0 +1,57 @@
+#this step depends on where you run
+#for CCin2p3
+site:
+    name: cc-parallel
+    mpi_command: "mpirun -n"
+
+#for NERSC
+#site:
+#    name: cori-batch
+#    image: ghcr.io/lsstdesc/txpipe-dev
+
+
+#all the following steps should not depend on where you run 
+launcher:
+    name: mini
+    interval: 3.0
+
+modules: >
+    txpipe
+    rail.estimation.algos.bpz_lite
+
+python_paths: []
+
+stages:
+#    - name: TXSourceSelectorMetadetect
+#      nprocess: 1
+#    - name: Inform_BPZ_lite
+#      nprocess: 1
+#    - name: BPZ_lite
+#      nprocess: 1
+    - name: CLClusterBinningRedshiftRichness
+      nprocess: 1
+#    - name: CLClusterShearCatalogs
+#      nprocess: 1
+#    - name: CLClusterEnsembleProfiles
+#      nprocess: 1
+#    - name: CLClusterDataVector
+#      nprocess: 1
+
+
+
+output_dir: ./data/cosmodc2/outputs-1deg2-CL
+config: examples/cosmodc2/config-1deg2-CL.yml
+
+inputs:
+    # See README for paths to download these files
+    #shear_catalog:  ./data/example/inputs/metadetect_shear_catalog.hdf5
+    #photometry_catalog:  ./data/example/inputs/photometry_catalog.hdf5
+    #fiducial_cosmology: ./data/fiducial_cosmology.yml
+    #calibration_table: ./data/example/inputs/sample_cosmodc2_w10year_errors.dat
+    #spectroscopic_catalog: ./data/example/inputs/mock_spectroscopic_catalog.hdf5
+    cluster_catalog: ./data/example/inputs/cluster_catalog.hdf5
+
+resume: True
+log_dir: ./data/cosmodc2/logs
+pipeline_log: ./data/cosmodc2/log_1deg2.txt
+

examples/cosmodc2/pipeline-1deg2-CL-nersc.yml

@@ -0,0 +1,57 @@
+#this step depends on where you run
+#for CCin2p3
+# site:
+#     name: cc-parallel
+#     mpi_command: "mpirun -n"
+
+#for NERSC
+site:
+   name: cori-batch
+   image: ghcr.io/lsstdesc/txpipe-dev
+
+
+#all the following steps should not depend on where you run 
+launcher:
+    name: mini
+    interval: 3.0
+
+modules: >
+    txpipe
+    rail.estimation.algos.bpz_lite
+
+python_paths: []
+
+stages:
+#    - name: TXSourceSelectorMetadetect
+#      nprocess: 1
+#    - name: Inform_BPZ_lite
+#      nprocess: 1
+#    - name: BPZ_lite
+#      nprocess: 1
+    - name: CLClusterBinningRedshiftRichness
+      nprocess: 1
+#    - name: CLClusterShearCatalogs
+#      nprocess: 1
+#    - name: CLClusterEnsembleProfiles
+#      nprocess: 1
+#    - name: CLClusterDataVector
+#      nprocess: 1
+
+
+
+output_dir: ./data/cosmodc2/outputs-1deg2-CL
+config: ./examples/cosmodc2/config-1deg2-CL.yml
+
+inputs:
+    # See README for paths to download these files
+#    shear_catalog:  ./data/example/inputs/metadetect_shear_catalog.hdf5
+#    photometry_catalog:  ./data/example/inputs/photometry_catalog.hdf5
+#    fiducial_cosmology: ./data/fiducial_cosmology.yml
+#    calibration_table: ./data/example/inputs/sample_cosmodc2_w10year_errors.dat
+#    spectroscopic_catalog: ./data/example/inputs/mock_spectroscopic_catalog.hdf5
+    cluster_catalog: ./data/example/inputs/cluster_catalog.hdf5
+
+resume: True
+log_dir: ./data/cosmodc2/logs
+pipeline_log: ./data/cosmodc2/log_1deg2.txt
+

examples/cosmodc2/pipeline-20deg2-CL-in2p3.yml

@@ -0,0 +1,56 @@
+#this step depends on where you run
+#for CCin2p3
+site:
+    name: cc-parallel
+    mpi_command: "mpirun -n"
+
+#for NERSC
+#site:
+#    name: cori-batch
+#    image: ghcr.io/lsstdesc/txpipe-dev
+
+
+#all the following steps should not depend on where you run 
+launcher:
+    name: mini
+    interval: 3.0
+modules: >
+    txpipe
+    rail.estimation.algos.bpz_lite
+
+python_paths: []
+
+stages:
+#    - name: TXSourceSelectorMetadetect
+#      nprocess: 30
+#    - name: Inform_BPZ_lite
+#      nprocess: 1
+#    - name: BPZ_lite
+#      nprocess: 30
+    - name: CLClusterBinningRedshiftRichness
+      nprocess: 1
+#    - name: CLClusterShearCatalogs
+#      nprocess: 30
+#    - name: CLClusterEnsembleProfiles
+#      nprocess: 30
+#    - name: CLClusterDataVector
+#      nprocess: 1
+
+
+
+output_dir: ./data/cosmodc2/outputs-20deg2-CL
+config: ./examples/cosmodc2/config-20deg2-CL.yml
+
+inputs:
+    # See README for paths to download these files
+    #shear_catalog:  ./data/cosmodc2/20deg2/shear_catalog.hdf5
+    #photometry_catalog:  ./data/cosmodc2/20deg2/photometry_catalog.hdf5
+    #fiducial_cosmology: ./data/fiducial_cosmology.yml
+    #calibration_table:  ./data/cosmodc2/20deg2/sample_cosmodc2_w10year_errors.dat
+    #spectroscopic_catalog: ./data/cosmodc2/20deg2/spectroscopic_catalog.hdf5
+    cluster_catalog: ./data/cosmodc2/20deg2/cluster_catalog.hdf5
+
+resume: True
+log_dir: ./data/cosmodc2/logs
+pipeline_log: ./data/cosmodc2/log_20deg2.txt
+

examples/cosmodc2/pipeline-20deg2-CL-nersc.yml

@@ -0,0 +1,56 @@
+#this step depends on where you run
+#for CCin2p3
+# site:
+#     name: cc-parallel
+#     mpi_command: "mpirun -n"
+
+#for NERSC
+site:
+   name: cori-batch
+   image: ghcr.io/lsstdesc/txpipe-dev
+
+
+#all the following steps should not depend on where you run 
+launcher:
+    name: mini
+    interval: 3.0
+modules: >
+    txpipe
+    rail.estimation.algos.bpz_lite
+
+python_paths: []
+
+stages:
+#    - name: TXSourceSelectorMetadetect
+#      nprocess: 30
+#    - name: Inform_BPZ_lite
+#      nprocess: 1
+#    - name: BPZ_lite
+#      nprocess: 30
+    - name: CLClusterBinningRedshiftRichness
+      nprocess: 1
+#    - name: CLClusterShearCatalogs
+#      nprocess: 30
+#    - name: CLClusterEnsembleProfiles
+#      nprocess: 1
+#    - name: CLClusterDataVector
+#      nprocess: 1
+
+
+
+output_dir: ./data/cosmodc2/outputs-20deg2-CL
+config: ./examples/cosmodc2/config-20deg2-CL.yml
+
+inputs:
+    # See README for paths to download these files
+    #shear_catalog:  ./data/cosmodc2/20deg2/shear_catalog.hdf5
+    #photometry_catalog:  ./data/cosmodc2/20deg2/photometry_catalog.hdf5
+    #fiducial_cosmology: ./data/fiducial_cosmology.yml
+    #calibration_table:  ./data/cosmodc2/20deg2/sample_cosmodc2_w10year_errors.dat
+    #spectroscopic_catalog: ./data/cosmodc2/20deg2/spectroscopic_catalog.hdf5
+    cluster_catalog: ./data/cosmodc2/20deg2/cluster_catalog.hdf5
+
+resume: True
+log_dir: ./data/cosmodc2/logs
+pipeline_log: ./data/cosmodc2/log_20deg2.txt
+