feat!: make AnalysisEpic compatible with production 22.11.2

.github/workflows/ci.yml

@@ -142,6 +142,7 @@ jobs:
       fail-fast: true
       matrix:
         include:
+          - { detector: epic,   num_files: 20 }
           - { detector: athena, num_files: 20 }
           - { detector: ecce,   num_files: 40 }
     steps:
@@ -241,11 +242,16 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        detector: [athena, ecce]
+        detector: [epic, athena, ecce]
         aname: [x_q2, p_eta, yRatio]
         recon: [Ele]
         include: # enable other recon methods for `aname==x_q2` only
-        - { aname: x_q2, recon: Mixed,  detector: athena } # FIXME: not sure how to avoid being repetitive... 
+        - { aname: x_q2, recon: Mixed,  detector: epic } # FIXME: not sure how to avoid being repetitive... 
+        - { aname: x_q2, recon: JB,     detector: epic }
+        - { aname: x_q2, recon: DA,     detector: epic }
+        - { aname: x_q2, recon: Sigma,  detector: epic }
+        - { aname: x_q2, recon: eSigma, detector: epic }
+        - { aname: x_q2, recon: Mixed,  detector: athena }
         - { aname: x_q2, recon: JB,     detector: athena }
         - { aname: x_q2, recon: DA,     detector: athena }
         - { aname: x_q2, recon: Sigma,  detector: athena }
@@ -295,6 +301,7 @@ jobs:
       matrix:
         mode:
           - fastsim
+          - epic
           - athena
           - ecce
         pname: # list only jobs which will only use one (primary) recon method
@@ -314,6 +321,12 @@ jobs:
           - { mode: fastsim, pname: coverage2D_x_q2, recon: JB,     aname: x_q2, pmacro: postprocess_x_q2.C }
           - { mode: fastsim, pname: coverage2D_x_q2, recon: Mixed,  aname: x_q2, pmacro: postprocess_x_q2.C }
           - { mode: fastsim, pname: coverage2D_x_q2, recon: Sigma,  aname: x_q2, pmacro: postprocess_x_q2.C }
+          - { mode: epic,    pname: coverage2D_x_q2, recon: Ele,    aname: x_q2, pmacro: postprocess_x_q2.C }
+          - { mode: epic,    pname: coverage2D_x_q2, recon: DA,     aname: x_q2, pmacro: postprocess_x_q2.C }
+          - { mode: epic,    pname: coverage2D_x_q2, recon: eSigma, aname: x_q2, pmacro: postprocess_x_q2.C }
+          - { mode: epic,    pname: coverage2D_x_q2, recon: JB,     aname: x_q2, pmacro: postprocess_x_q2.C }
+          - { mode: epic,    pname: coverage2D_x_q2, recon: Mixed,  aname: x_q2, pmacro: postprocess_x_q2.C }
+          - { mode: epic,    pname: coverage2D_x_q2, recon: Sigma,  aname: x_q2, pmacro: postprocess_x_q2.C }
           - { mode: athena,  pname: coverage2D_x_q2, recon: Ele,    aname: x_q2, pmacro: postprocess_x_q2.C }
           - { mode: athena,  pname: coverage2D_x_q2, recon: DA,     aname: x_q2, pmacro: postprocess_x_q2.C }
           - { mode: athena,  pname: coverage2D_x_q2, recon: eSigma, aname: x_q2, pmacro: postprocess_x_q2.C }
@@ -403,6 +416,10 @@ jobs:
         with:
           name: root_analysis_fastsim_${{matrix.aname}}_${{matrix.recon}}
           path: out
+      - uses: actions/download-artifact@v3
+        with:
+          name: root_analysis_epic_${{matrix.aname}}_${{matrix.recon}}
+          path: out
       - uses: actions/download-artifact@v3
         with:
           name: root_analysis_athena_${{matrix.aname}}_${{matrix.recon}}
@@ -426,10 +443,12 @@ jobs:
             echo "[CI] COMPARATOR MACRO"
             ls out
             mv -v out/fastsim.{${{matrix.aname}},${{matrix.pname}}}.${{matrix.recon}}.root # rename aname -> pname
+            mv -v out/epic.{${{matrix.aname}},${{matrix.pname}}}.${{matrix.recon}}.root # rename aname -> pname
             mv -v out/athena.{${{matrix.aname}},${{matrix.pname}}}.${{matrix.recon}}.root # rename aname -> pname
             mv -v out/ecce.{${{matrix.aname}},${{matrix.pname}}}.${{matrix.recon}}.root # rename aname -> pname
-            ${{env.root_no_delphes}} 'macro/ci/comparator.C("out/fastsim.${{matrix.pname}}.${{matrix.recon}}.root","out/athena.${{matrix.pname}}.${{matrix.recon}}.root","out/ecce.${{matrix.pname}}.${{matrix.recon}}.root","out/comparison.${{matrix.pname}}.${{matrix.recon}}","${{matrix.xvar}}","${{matrix.yvar}}")'
+            ${{env.root_no_delphes}} 'macro/ci/comparator.C("out/fastsim.${{matrix.pname}}.${{matrix.recon}}.root","out/epic.${{matrix.pname}}.${{matrix.recon}}.root","out/athena.${{matrix.pname}}.${{matrix.recon}}.root","out/ecce.${{matrix.pname}}.${{matrix.recon}}.root","out/comparison.${{matrix.pname}}.${{matrix.recon}}","${{matrix.xvar}}","${{matrix.yvar}}")'
             rm -v out/fastsim.${{matrix.pname}}.${{matrix.recon}}.root # rm analysis_root artifact
+            rm -v out/epic.${{matrix.pname}}.${{matrix.recon}}.root # rm analysis_root artifact
             rm -v out/athena.${{matrix.pname}}.${{matrix.recon}}.root # rm analysis_root artifact
             rm -v out/ecce.${{matrix.pname}}.${{matrix.recon}}.root # rm analysis_root artifact
       - uses: actions/upload-artifact@v3

.gitignore

@@ -25,6 +25,7 @@ results*
 # tmp files
 tutorial/delphes.config
 tutorial/delphes.config.bak
+tutorial/s3files.*.config
 get-files.sh
 *.xsec
 

datarec/xsec/xsec.dat

@@ -1,4 +1,32 @@
+# Pythia 6, for EPIC: S3/eictest/EPIC/EVGEN/DIS/NC
+# qbins from 1-10-100-1000-100000 
+# noradcor is without radgen, radcor is including radgen 
+#label                                    cross_section_[pb] relative_uncertainty
+pythia6:ep_noradcor.18x275_q2_1_10        8.089e+05          0.0010      # 20   40.0 M 4.95e+01 nb-1
+pythia6:ep_noradcor.18x275_q2_10_100      7.087e+04          0.0007      # 20   20.0 M 2.82e+02 nb-1
+pythia6:ep_noradcor.18x275_q2_100_1000    3.034e+03          0.0347      # 40   4.0  M 1.32e+03 nb-1
+pythia6:ep_noradcor.18x275_q2_1000_100000 5.697e+01          0.0018      # 20   1.0  M 1.76e+04 nb-1
+pythia6:ep_noradcor.10x100_q2_1_10        5.387e+05          0.0015      # 20   40.0 M 7.42e+01 nb-1
+pythia6:ep_noradcor.10x100_q2_10_100      3.964e+04          0.0005      # 20   20.0 M 5.05e+02 nb-1
+pythia6:ep_noradcor.10x100_q2_100_1000    1.196e+03          0.0819      # 20   2.0  M 1.67e+03 nb-1
+pythia6:ep_noradcor.10x100_q2_1000_100000 4.286e+00          0.0104      # 20   1.0  M 2.33e+05 nb-1
+pythia6:ep_noradcor.5x100_q2_1_10         4.462e+05          0.0010      # 20   40.0 M 8.96e+01 nb-1
+pythia6:ep_noradcor.5x100_q2_10_100       2.902e+04          0.0291      # 20   20.0 M 6.89e+02 nb-1
+pythia6:ep_noradcor.5x100_q2_100_1000     6.471e+02          0.0146      # 20   2.0  M 3.09e+03 nb-1
+pythia6:ep_noradcor.5x100_q2_1000_100000  2.092e-01          0.0245      # 20   0.2  M 9.56e+05 nb-1
+pythia6:ep_noradcor.5x41_q2_1_10          3.433e+05          0.0015      # 20   40.0 M 1.17e+02 nb-1
+pythia6:ep_noradcor.5x41_q2_10_100        1.935e+04          0.0006      # 20   20.0 M 1.03e+03 nb-1
+pythia6:ep_noradcor.5x41_q2_100_1000      2.219e+02          0.0074      # 20   2.0  M 9.01e+03 nb-1
+pythia6:ep_radcor.18x275_q2_1_10          8.540e+05          0.0004      # 20   40.0 M 4.68e+01 nb-1 4.68e+01
+pythia6:ep_radcor.18x275_q2_10_100        1.455e+05          0.1198      # 20   20.0 M 1.37e+02 nb-1 1.38e+02
+pythia6:ep_radcor.18x275_q2_100_1000      6.919e+03          0.0483      # 40   4.0  M 5.78e+02 nb-1 5.78e+02
+pythia6:ep_radcor.18x275_q2_1000_100000   1.212e+02          0.0460      # 20   1.0  M 8.25e+03 nb-1 8.25e+03
+pythia6:ep_radcor.5x41_q2_1_10            3.730e+05          0.0740      # 200  20.0 M 5.36e+01 nb-1 5.36e+01
+pythia6:ep_radcor.5x41_q2_10_100          2.286e+04          0.2940      # 1000 10.0 M 4.37e+02 nb-1 4.37e+02
+pythia6:ep_radcor.5x41_q2_100_1000        2.576e+02          0.0074      # 20   2.0  M 7.76e+03 nb-1 7.76e+03
+
 # Pythia 8, from ATHENA production HEPMC files: S3/eictest/ATHENA/EVGEN/DIS/NC
+# Q2 binning by minimum only (no maxima)
 #label                     cross_section_[pb]  relative_uncertainty
 pythia8:5x100/minQ2=1000   0.43023             0.00258
 pythia8:5x100/minQ2=100    778.99              0.00223
@@ -19,8 +47,8 @@ pythia8:18x275/minQ2=1000  79.451              0.00223
 pythia8:18x275/minQ2=100   3370.2              0.00245
 pythia8:18x275/minQ2=10    69275               0.00266
 pythia8:18x275/minQ2=1     7.4167e+05          0.00277
-# Pythia 6, from ECCE production files:
-# S3/eictest/ECCE/ProductionInputFiles/SIDIS/pythia6
+
+# Pythia 6, from ECCE production files: S3/eictest/ECCE/ProductionInputFiles/SIDIS/pythia6
 # Note: the following are values for noradcor files, radcor 
 #label                     cross_section_[pb]  relative_uncertainty
 pythia6:ep-5x41            3.189e+05           0.0011   # general Q2

macro/ci/analysis_p_eta.C

@@ -8,11 +8,13 @@ void analysis_p_eta(
     TString configFile,
     TString outfilePrefix,
     TString reconMethod="Ele"
-) {
+    )
+{
 
   // setup analysis ========================================
   Analysis *A;
-  if     (outfilePrefix.Contains("athena")) A = new AnalysisAthena( configFile, outfilePrefix );
+  if   (outfilePrefix.Contains("epic"))     A = new AnalysisEpic(   configFile, outfilePrefix );
+  else if(outfilePrefix.Contains("athena")) A = new AnalysisAthena( configFile, outfilePrefix );
   else if(outfilePrefix.Contains("ecce"))   A = new AnalysisEcce(   configFile, outfilePrefix );
 #ifndef EXCLUDE_DELPHES
   else A = new AnalysisDelphes( configFile, outfilePrefix );

macro/ci/analysis_x_q2.C

@@ -13,7 +13,8 @@ void analysis_x_q2(
 
   // setup analysis ========================================
   Analysis *A;
-  if     (outfilePrefix.Contains("athena")) A = new AnalysisAthena( configFile, outfilePrefix );
+  if   (outfilePrefix.Contains("epic"))     A = new AnalysisEpic(   configFile, outfilePrefix );
+  else if(outfilePrefix.Contains("athena")) A = new AnalysisAthena( configFile, outfilePrefix );
   else if(outfilePrefix.Contains("ecce"))   A = new AnalysisEcce(   configFile, outfilePrefix );
 #ifndef EXCLUDE_DELPHES
   else A = new AnalysisDelphes( configFile, outfilePrefix );

macro/ci/analysis_yRatio.C

@@ -8,11 +8,13 @@ void analysis_yRatio(
     TString configFile,
     TString outfilePrefix,
     TString reconMethod="Ele"
-) {
+    )
+{
 
   // setup analysis ========================================
   Analysis *A;
-  if     (outfilePrefix.Contains("athena")) A = new AnalysisAthena( configFile, outfilePrefix );
+  if   (outfilePrefix.Contains("epic"))     A = new AnalysisEpic(   configFile, outfilePrefix );
+  else if(outfilePrefix.Contains("athena")) A = new AnalysisAthena( configFile, outfilePrefix );
   else if(outfilePrefix.Contains("ecce"))   A = new AnalysisEcce(   configFile, outfilePrefix );
 #ifndef EXCLUDE_DELPHES
   else A = new AnalysisDelphes( configFile, outfilePrefix );

macro/ci/comparator.C

@@ -7,8 +7,9 @@ R__LOAD_LIBRARY(Sidis-eic)
 // - depending on infile, different histograms will be drawn
 void comparator(
     TString infile0="out/resolution.fastsim.root",
-    TString infile1="out/resolution.athena.root",
-    TString infile2="out/resolution.ecce.root",
+    TString infile1="out/resolution.epic.root",
+    TString infile2="out/resolution.athena.root",
+    TString infile3="out/resolution.ecce.root",
     TString outfile="out/resolution.comparison.root",
     TString gx="x", TString gy="q2" // plotgrid vars
     ) {
@@ -76,7 +77,8 @@ void comparator(
   std::vector<TFile*> infiles = {
     new TFile(infile0),
     new TFile(infile1),
-    new TFile(infile2)
+    new TFile(infile2),
+    new TFile(infile3)
   };
   bool first=true;
   Int_t numXbins, numYbins;
@@ -117,9 +119,9 @@ void comparator(
   // set legend labels
   auto makeLegendName = [] (TString infileName) -> TString {
     if     (infileName.Contains("fastsim")) return "Delphes";
+    else if(infileName.Contains("epic"))    return "EPIC";
     else if(infileName.Contains("athena"))  return "ATHENA";
     else if(infileName.Contains("ecce"))    return "ECCE";
-    else if(infileName.Contains("epic"))    return "EPIC";
     return "UNKNOWN";
   };
   for(auto infile : infiles) {

s3tools/download.sh

@@ -23,7 +23,12 @@ while read sourceURL; do
   sourcePath=$(echo $sourceURL | awk '{print $1}' | sed 's/https:\/\///g' | sed 's/^[^\/]*\//S3\//g')
   echo "mc cp $sourcePath $targetPath; echo \"\"" >> $downloadScript
 done
-echo "generated downloader script $downloadScript"
+echo """
+generated downloader script $downloadScript
+===============================================================
+`cat $downloadScript`
+===============================================================
+"""
 
 # download
 echo "now starting download..."

s3tools/make-epic-config.sh

@@ -0,0 +1,121 @@
+#!/bin/bash
+
+# SPDX-License-Identifier: LGPL-3.0-or-later
+# Copyright (C) 2022 Christopher Dilks
+
+
+###################
+# TOP-LEVEL SCRIPT to automate the creation of a config file for a specific release,
+# supporting streaming or downloading from S3
+# - the config file consists of file names (or URLs), with Q2 minima and cross sections
+###################
+
+# RELEASE TAG AND RECO DIR: ###########################
+detector_config="epic_arches"
+# detector_config="epic_brycecanyon"
+release="22.11.2"
+releaseDir="S3/eictest/EPIC/RECO/$release/$detector_config/DIS/NC"
+filter='eicrecon'
+# filter='juggler'
+#######################################################
+
+# usage:
+if [ $# -lt 3 ]; then
+  echo """
+  USAGE: $0 [energy] [local_dir] [mode] [limit(optional)] [config_file(optional)]
+
+   - [energy]: 5x41      | - see below for available datasets
+               5x100     | - data from different Q2minima are combined,
+               10x100    |   weighted by cross sections
+               10x275
+               18x275
+
+   - [local_dir]: output directory name: datarec/[local_dir]
+
+   - [mode]:   s - make config file for streaming from S3
+               d - download from S3, then make the local config file
+               c - just make the local config file, for local files
+
+   - [limit]   integer>0 : only stream/download this many files per Q2 min
+               0         : stream/download all files
+               default=5
+
+   - [config_file]  name of the config file; if not specified, the
+                    config file will be in datarec/[local_dir]
+
+   See script for local and remote file path settings; they are
+   configured for a specific set of data, but you may want to change
+   them.
+
+  CURRENT RELEASE: $release
+  S3 Directory:    $releaseDir
+
+  AVAILABLE DATA ON S3 (press ^C to abort S3 query):
+  """
+  mc tree $releaseDir
+  exit 2
+fi
+energy=$1
+locDir=$2
+mode=$3
+limit=5
+configFile=""
+if [ $# -ge 4 ]; then limit=$4; fi
+if [ $# -ge 5 ]; then configFile=$5; fi
+
+# cd to the main directory 
+pushd $(dirname $(realpath $0))/..
+
+# settings #############################################################
+sourceDir="$releaseDir/$energy"
+targetDir="datarec/$locDir/$release/$energy"
+Q2minima=( 1000 100  10  1  )
+Q2maxima=( 0    1000 100 10 )
+########################################################################
+
+# download files from S3
+function q2subdir { echo $1/minQ2=$2; }
+function status { echo ""; echo "[+] $1"; }
+if [ "$mode" == "d" ]; then
+  status "downloading files from S3..."
+  for Q2min in ${Q2minima[@]}; do
+    echo " sourceDir = `q2subdir $sourceDir $Q2min`"
+    echo " targetDir = `q2subdir $targetDir $Q2min`"
+    s3tools/generate-s3-list.sh  `q2subdir $sourceDir $Q2min`   | \
+      { if [ $limit -gt 0 ]; then head -n$limit; else cat; fi } | \
+      grep -E $filter                                           | \
+      s3tools/download.sh `q2subdir $targetDir $Q2min`
+  done
+fi
+
+# build a config file
+status "build config file..."
+mkdir -p $targetDir
+if [ -z "$configFile" ]; then configFile=$targetDir/files.config; fi
+> $configFile.list
+for (( i=0; i<${#Q2minima[@]}; i++)); do
+  Q2min=${Q2minima[$i]}
+  Q2max=${Q2maxima[$i]}
+  crossSection=$(s3tools/read-xsec-table.sh "pythia8:$energy/minQ2=$Q2min")
+  case $mode in
+    d) listScript=s3tools/generate-local-list.sh; listDir=`q2subdir $targetDir $Q2min`; ;;
+    c) listScript=s3tools/generate-local-list.sh; listDir=`q2subdir $targetDir $Q2min`; ;;
+    s) listScript=s3tools/generate-s3-list.sh;    listDir=`q2subdir $sourceDir $Q2min`; ;;
+    *) echo "ERROR: unknown mode" >&2; exit 1;    ;;
+  esac
+  $listScript $listDir $crossSection $Q2min $Q2max            | \
+    grep -v UNKNOWN                                           | \
+    { if [ $limit -gt 0 ]; then head -n$limit; else cat; fi } | \
+    tee -a $configFile.list
+done
+s3tools/generate-config-file.rb $configFile $energy $configFile.list
+
+# finalize
+popd
+status "done building config file at:"
+echo "     $configFile"
+echo ""
+if [ -n "$(grep UNKNOWN $configFile.list)" ]; then
+  >&2 echo "ERROR: missing some cross sections"
+  exit 1
+fi

src/Analysis.cxx

@@ -94,6 +94,7 @@ Analysis::Analysis(
   // - these settings can be set at the macro level
   verbose         = false;
   writeSimpleTree = false;
+  writeParticleTree = false;
   maxEvents       = 0;
   useBreitJets    = false;
   errorCntMax     = 1000;
@@ -310,6 +311,7 @@ void Analysis::Prepare() {
   kin = new Kinematics(eleBeamEn,ionBeamEn,crossingAngle);
   kinTrue = new Kinematics(eleBeamEn, ionBeamEn, crossingAngle);
   ST = new SimpleTree("tree",kin,kinTrue);
+  PT = new ParticleTree("ptree");
 
   // if including jets, define a `jet` final state
 #ifndef EXCLUDE_DELPHES
@@ -659,6 +661,7 @@ void Analysis::Finish() {
   cout << "writing ROOT file..." << endl;
   outFile->cd();
   if(writeSimpleTree) ST->WriteTree();
+  if(writeParticleTree) PT->WriteTree();
   HD->Payload([this](Histos *H){ H->WriteHists(outFile); }); HD->ExecuteAndClearOps();
   HD->Payload([this](Histos *H){ H->Write(); }); HD->ExecuteAndClearOps();
   std::vector<Double_t> vec_wInclusiveTotal { wInclusiveTotal };

src/Analysis.h

@@ -28,10 +28,12 @@
 #include "adage/BinSet.h"
 
 // sidis-eic
+#include "DataModel.h"
 #include "Histos.h"
 #include "HistosDAG.h"
 #include "Kinematics.h"
 #include "SimpleTree.h"
+#include "ParticleTree.h"
 #include "Weights.h"
 #include "CommonConstants.h"
 
@@ -59,6 +61,7 @@ class Analysis : public TNamed
     // common settings
     Bool_t verbose; // if true, print a lot more information
     Bool_t writeSimpleTree; // if true, write SimpleTree (not binned)
+    Bool_t writeParticleTree; // if true, write ParticleTree (not binned)
     Long64_t maxEvents; /* default=0, which runs all events;
                          * if > 0, run a maximum number of `maxEvents` events (useful for quick tests)
                          */
@@ -121,6 +124,7 @@ class Analysis : public TNamed
 
     // shared objects
     SimpleTree *ST;
+    ParticleTree *PT;
     Kinematics *kin, *kinTrue;
     HistosDAG *HD;
     Weights const* weightInclusive;