From 315b268f8fae498613aa3f70d7f4e44e73dd53d8 Mon Sep 17 00:00:00 2001
From: IzaakWN <iwn_@hotmail.com>
Date: Mon, 22 Apr 2024 19:31:46 +0200
Subject: [PATCH 01/10] add domutau flag to avoid error for non-mutau channels
 / non-DY samples

---
 PicoProducer/python/analysis/ModuleMuTau.py   |  8 ++--
 PicoProducer/python/analysis/ModuleTauPair.py | 45 ++++++++++---------
 .../python/analysis/TreeProducerMuTau.py      |  2 +-
 3 files changed, 29 insertions(+), 26 deletions(-)

diff --git a/PicoProducer/python/analysis/ModuleMuTau.py b/PicoProducer/python/analysis/ModuleMuTau.py
index 2d8d34c57..088dcf57b 100644
--- a/PicoProducer/python/analysis/ModuleMuTau.py
+++ b/PicoProducer/python/analysis/ModuleMuTau.py
@@ -53,14 +53,14 @@ def __init__(self, fname, **kwargs):
     self.out.cutflow.addcut('pair',         "pair"                       )
     self.out.cutflow.addcut('weight',       "no cut, weighted", 15       )
     self.out.cutflow.addcut('weight_no0PU', "no cut, weighted, PU>0", 16 ) # use for normalization; bug in pre-UL 2017 caused small fraction of events with nPU<=0
-    ## Important cutflow entries to make stitching with exclusive mutauh sample
+    # Important cutflow entries to make stitching with exclusive mutauh sample (DYJetsToTauTauToMuTauh)
     self.out.cutflow.addcut('weight_mutaufilter', "no cut, mutaufilter", 17 )    
     self.out.cutflow.addcut('weight_mutaufilter_NUP0orp4', "no cut, weighted, mutau, 0 or >4 jets", 18 )
     self.out.cutflow.addcut('weight_mutaufilter_NUP1', "no cut, weighted, mutau, 1 jet", 19 )
     self.out.cutflow.addcut('weight_mutaufilter_NUP2', "no cut, weighted, mutau, 2 jets", 20 )
     self.out.cutflow.addcut('weight_mutaufilter_NUP3', "no cut, weighted, mutau, 3 jets", 21 )
     self.out.cutflow.addcut('weight_mutaufilter_NUP4', "no cut, weighted, mutau, 4 jets", 22 )
-
+    
   
   def beginJob(self):
     """Before processing any events or files."""
@@ -244,8 +244,8 @@ def analyze(self, event):
       self.out.genvistaueta_2[0] = eta
       self.out.genvistauphi_2[0] = phi
       self.out.gendm_2[0]        = status
-      if self.dozpt:
-        self.out.mutaufilter[0]  = filtermutau(event) # for stitching DYJetsToTauTauToMuTauh
+      if self.domutau:
+        self.out.mutaufilter[0]  = self.ismutau #filtermutau(event) # for stitching DYJetsToTauTauToMuTauh
     
     
     # JETS
diff --git a/PicoProducer/python/analysis/ModuleTauPair.py b/PicoProducer/python/analysis/ModuleTauPair.py
index e2323b837..65ecb407f 100644
--- a/PicoProducer/python/analysis/ModuleTauPair.py
+++ b/PicoProducer/python/analysis/ModuleTauPair.py
@@ -44,6 +44,7 @@ def __init__(self, fname, **kwargs):
     self.tauwp      = kwargs.get('tauwp',    1              ) # minimum DeepTau WP, e.g. 1 = VVVLoose, etc.
     self.dotoppt    = kwargs.get('toppt',    'TT' in fname  ) # top pT reweighting
     self.dozpt      = kwargs.get('zpt',      'DY' in fname  ) # Z pT reweighting
+    self.domutau    = kwargs.get('domutau',  'DY' in fname or self.dozpt ) # mutau genfilter for stitching DY sample
     self.dopdf      = kwargs.get('dopdf',    False          ) and self.ismc # store PDF & scale weights
     self.dorecoil   = kwargs.get('recoil',   False          ) and self.ismc # recoil corrections #('DY' in name or re.search(r"W\d?Jets",name)) and self.year==2016) # and self.year==2016 
     self.dosys      = self.tessys in [None,''] and self.ltf in [1,None] and self.jtf in [1,None] # include systematic variations of weight
@@ -66,6 +67,7 @@ def __init__(self, fname, **kwargs):
     self.filter     = getmetfilters(self.era,self.isdata,verb=self.verbosity)
     
     # CORRECTIONS
+    self.ismutau      = False # event passes gen mutau filter (to avoid computing twice)
     self.ptnom        = lambda j: j.pt # use 'pt' as nominal jet pt (not corrected)
     self.jecUncLabels = [ ]
     self.metUncLabels = [ ]
@@ -111,6 +113,7 @@ def beginJob(self):
     print(">>> %-12s = %s"%('dotoppt',   self.dotoppt))
     print(">>> %-12s = %s"%('dopdf',     self.dopdf))
     print(">>> %-12s = %s"%('dozpt',     self.dozpt))
+    print(">>> %-12s = %s"%('domutau',   self.domutau))
     #print ">>> %-12s = %s"%('dorecoil',  self.dorecoil)
     print(">>> %-12s = %s"%('dojec',     self.dojec))
     print(">>> %-12s = %s"%('dojecsys',  self.dojecsys))
@@ -150,9 +153,8 @@ def beginFile(self, inputFile, outputFile, inputTree, wrappedOutputTree):
       ('Tau_idDeepTau2018v2p5VSjet','Tau_idDeepTau2017v2p1VSjet'),
       ('Tau_rawDeepTau2018v2p5VSe','Tau_rawDeepTau2017v2p1VSe'), 
       ('Tau_rawDeepTau2018v2p5VSmu','Tau_rawDeepTau2017v2p1VSmu'),  
-      ('Tau_rawDeepTau2018v2p5VSjet','Tau_rawDeepTau2017v2p1VSjet') 
- 
-      ]
+      ('Tau_rawDeepTau2018v2p5VSjet','Tau_rawDeepTau2017v2p1VSjet')
+    ]
     # for v9
     branches = [
       ('Electron_mvaFall17V2Iso',        'Electron_mvaFall17Iso'        ),
@@ -173,8 +175,7 @@ def beginFile(self, inputFile, outputFile, inputTree, wrappedOutputTree):
       ('Tau_idDeepTau2018v2p5VSjet','Tau_idDeepTau2017v2p1VSjet'),
       ('Tau_rawDeepTau2018v2p5VSe','Tau_rawDeepTau2017v2p1VSe'), 
       ('Tau_rawDeepTau2018v2p5VSmu','Tau_rawDeepTau2017v2p1VSmu'),  
-      ('Tau_rawDeepTau2018v2p5VSjet','Tau_rawDeepTau2017v2p1VSjet') 
-
+      ('Tau_rawDeepTau2018v2p5VSjet','Tau_rawDeepTau2017v2p1VSjet')
     ]
     if self.year==2016:
       branches += [
@@ -219,22 +220,24 @@ def fillhists(self,event):
         self.out.cutflow.fill('weight_no0PU',event.genWeight)
       else: # bug in pre-UL 2017 caused small fraction of events with nPU<=0
         return False
-      # Specific selections to compute mutau filter efficiencies for stitching of different DY samples
-      isMuTau = filtermutau(event)
-      self.out.cutflow.fill('weight_mutaufilter',event.genWeight*isMuTau)
-      try:
-        if event.LHE_Njets==0 or event.LHE_Njets>4:
-          self.out.cutflow.fill('weight_mutaufilter_NUP0orp4',event.genWeight*isMuTau)
-        elif event.LHE_Njets==1:
-          self.out.cutflow.fill('weight_mutaufilter_NUP1',event.genWeight*isMuTau)
-        elif event.LHE_Njets==2:
-          self.out.cutflow.fill('weight_mutaufilter_NUP2',event.genWeight*isMuTau)
-        elif event.LHE_Njets==3:
-          self.out.cutflow.fill('weight_mutaufilter_NUP3',event.genWeight*isMuTau)
-        elif event.LHE_Njets==4:
-          self.out.cutflow.fill('weight_mutaufilter_NUP4',event.genWeight*isMuTau)
-      except RuntimeError:
-        no_LHE_Njets_var = True
+      # Specific selections to compute mutau filter efficiencies for stitching of different DY samples (DYJetsToTauTauToMuTauh)
+      if self.domutau:
+        self.ismutau = filtermutau(event) # event passes gen mutau filter
+        self.out.cutflow.fill('weight_mutaufilter',event.genWeight*isMuTau)
+        try:
+          if event.LHE_Njets==0 or event.LHE_Njets>4:
+            self.out.cutflow.fill('weight_mutaufilter_NUP0orp4',event.genWeight*isMuTau)
+          elif event.LHE_Njets==1:
+            self.out.cutflow.fill('weight_mutaufilter_NUP1',event.genWeight*isMuTau)
+          elif event.LHE_Njets==2:
+            self.out.cutflow.fill('weight_mutaufilter_NUP2',event.genWeight*isMuTau)
+          elif event.LHE_Njets==3:
+            self.out.cutflow.fill('weight_mutaufilter_NUP3',event.genWeight*isMuTau)
+          elif event.LHE_Njets==4:
+            self.out.cutflow.fill('weight_mutaufilter_NUP4',event.genWeight*isMuTau)
+        except RuntimeError:
+          print(">>> WARNING: RuntimeError! Setting domutau=False !")
+          self.domutau = False
       self.out.pileup.Fill(event.Pileup_nTrueInt)
     
     return True
diff --git a/PicoProducer/python/analysis/TreeProducerMuTau.py b/PicoProducer/python/analysis/TreeProducerMuTau.py
index 721db5cf0..42e4a1d77 100644
--- a/PicoProducer/python/analysis/TreeProducerMuTau.py
+++ b/PicoProducer/python/analysis/TreeProducerMuTau.py
@@ -101,6 +101,6 @@ def __init__(self, filename, module, **kwargs):
         self.addBranch('idweightDown_dm_2', 'f', 1.)
         self.addBranch('ltfweightUp_2',     'f', 1.)
         self.addBranch('ltfweightDown_2',   'f', 1.)
-      if self.module.dozpt:
+      if self.module.domutau:
         self.addBranch('mutaufilter',       '?', title="has tautau -> mutau, pT>18, |eta|<2.5")
     

From 4c1fcf2fed9deef701c29732450cc74e41907989 Mon Sep 17 00:00:00 2001
From: IzaakWN <iwn_@hotmail.com>
Date: Mon, 22 Apr 2024 19:30:27 +0200
Subject: [PATCH 02/10] move from uberFTP to gFal tools to support EOS outside
 lxplus (T3 PSI, UCL)

---
 PicoProducer/python/storage/EOS.py | 29 ++++++++++-------------------
 1 file changed, 10 insertions(+), 19 deletions(-)

diff --git a/PicoProducer/python/storage/EOS.py b/PicoProducer/python/storage/EOS.py
index f1c2d3ee9..78d4f6b8a 100644
--- a/PicoProducer/python/storage/EOS.py
+++ b/PicoProducer/python/storage/EOS.py
@@ -3,14 +3,14 @@
 # https://cern.service-now.com/service-portal?id=kb_article&sys_id=fae8543fc9ed05006d218776d679b74a
 import os
 from TauFW.common.tools.utils import execute
+from TauFW.PicoProducer.storage.utils import host
 from TauFW.PicoProducer.storage.StorageSystem import StorageSystem
-import getpass, platform
 
 
 class EOS(StorageSystem):
   
   def __init__(self,path,verb=0,ensure=False,eos=False,**kwargs):
-    """EOS is mounted on lxplus, so no special override are necessary."""
+    """EOS is mounted on lxplus, so no special overload are necessary."""
     super(EOS,self).__init__(path,verb=verb,ensure=False,**kwargs)
     if not self.mounted: # EOS is mounted on lxplus
       if eos: # use EOS command
@@ -18,15 +18,15 @@ def __init__(self,path,verb=0,ensure=False,eos=False,**kwargs):
         os.environ["EOS_MGM_URL"] = "root://eosuser.cern.ch"
         self.lscmd = "eos ls" # first do export EOS_MGM_URL=root://eosuser.cern.ch
         self.lscmd = "eos rm" # first do export EOS_MGM_URL=root://eosuser.cern.ch
-      else: # use uberftp; NOTE: doest not work for /eos/user/...
-        #self.lscmd = "uberftp -ls" # stopped working 12/2023
-        #self.lsurl = "gsiftp://eoscmsftp.cern.ch/"
-        self.lscmd = "LD_LIBRARY_PATH='' PYTHONPATH='' gfal-ls -l"
+      else: # NOTE: uberftp no longer supported for EOS...
+        unset = "" if 'ucl' in host else "LD_LIBRARY_PATH='' PYTHONPATH='' " # unset libraries that break gFal tools
+        self.lscmd = unset+"gfal-ls -l"
         self.lsurl = "root://eoscms.cern.ch/"
         self.lscol = -1 # take last column
-        self.rmcmd = 'uberftp -rm'
-        self.rmurl = 'gsiftp://eoscmsftp.cern.ch/'
-        self.mkdir = self._mkdir # override default StorageSystem.mkdir
+        self.mkdrcmd = unset+"gfal-mkdir -p"
+        self.mkdrurl = "root://eosuser.cern.ch/"
+        self.rmcmd = unset+"gfal-rm -r"
+        self.rmurl = "root://eosuser.cern.ch/"
       self.cpcmd   = 'xrdcp -f'
       self.chmdprm = '2777'
       self.cpurl   = "root://eoscms.cern.ch/"
@@ -35,13 +35,4 @@ def __init__(self,path,verb=0,ensure=False,eos=False,**kwargs):
     self.tmpdir    = '/tmp/$USER/'
     if ensure:
       self.ensuredir(self.path)
-  
-  #def _rm(self,*paths,**kwargs):
-  #  path = self.expandpath(*paths,here=True)
-  #  verb = kwargs.get('verb',self.verbosity)
-  #  return self.execute("uberftp storage01.lcg.cscs.ch 'rm -r %s'"%(path),verb=verb)
-  
-  def _mkdir(self,dirname='$PATH',**kwargs):
-    verb    = kwargs.get('verb',self.verbosity)
-    dirname = self.expandpath(dirname,here=True)
-    return self.execute("uberftp eoscmsftp.cern.ch 'mkdir %s'"%(dirname),verb=verb)
\ No newline at end of file
+    
\ No newline at end of file

From bbd503c9e661ed2a361cbff39a0689b60b0d7d64 Mon Sep 17 00:00:00 2001
From: IzaakWN <iwn_@hotmail.com>
Date: Mon, 22 Apr 2024 19:29:26 +0200
Subject: [PATCH 03/10] resolve conflicts; fix lxplus HTCondor environment
 issue; add possibility for CMSSW singularity

---
 PicoProducer/python/batch/submit_HTCondor.sh  | 87 +++++++++++++------
 PicoProducer/python/batch/submit_HTCondor.sub | 15 +++-
 .../python/batch/submit_HTCondor_CMG.sub      | 15 +++-
 3 files changed, 85 insertions(+), 32 deletions(-)

diff --git a/PicoProducer/python/batch/submit_HTCondor.sh b/PicoProducer/python/batch/submit_HTCondor.sh
index 7073e4582..86cde56ed 100755
--- a/PicoProducer/python/batch/submit_HTCondor.sh
+++ b/PicoProducer/python/batch/submit_HTCondor.sh
@@ -2,39 +2,76 @@
 ## Script to run on a HTCondor batch system
 
 # START
-START=`date +%s`
-echo "Job start at `date`"
-echo "Running job on machine `uname -a`, host $HOSTNAME"
-function peval { echo ">>> $@"; eval "$@"; }
+START="$(date +%s)"
+echo "Job starts at $(date)"
+echo "Running job on machine $(uname -a), host $HOSTNAME"
 
-# SETTING
-TASKCMD="$@"
-WORKDIR="$PWD"
-printf '=%.0s' `seq 60`; echo
-echo "\$PWD=$PWD"
-echo "\$JOBID=$JOBID"
-echo "\$TASKID=$TASKID"
-echo "\$HOSTNAME=$HOSTNAME"
-echo "\$TASKCMD=$TASKCMD"
-echo "\$WORKDIR=$WORKDIR"
-#printf '=%.0s' `seq 60`; echo
-#env
-#printf '=%.0s' `seq 60`; echo
+# SETTINGS: store in file for reuse
+cat << EOF > setenv.sh
+function peval { echo ">>> \$@"; eval "\$@"; }
+function pbar { printf '=%.0s' \$(seq \${1:-70}); echo; }
+VERB=${VERB:-0} # verbosity level for debugging
+START=$START
+JOBID=$JOBID
+TASKID=$TASKID
+HOSTNAME=$HOSTNAME
+PWD=$PWD
+WORKDIR=$PWD
+CONTAINER='$CONTAINER' # to set OS environment with container (Singularity, e.g. "cmssw-el7")
+CMSSW_BASE=$CMSSW_BASE # to set CMSSW environment
+TASKCMD='$@'
+EOF
+source setenv.sh # set environment
+pbar
+peval 'tail -n +3 setenv.sh | while read line; do echo "\$$line"; done'
+pbar
+[ $VERB -ge 1 ] && { peval "env"; pbar; } # print out environment for debugging
+
+# OS ENVIRONMENT with container (Singularity)
+# https://cms-sw.github.io/singularity.html
+# https://apptainer.org/docs/user/main/environment_and_metadata.html
+if [ ! -z "$CONTAINER" ]; then # if $CONTAINER is set
+  echo ">>> Setting OS environment with container '$CONTAINER'..."
+  peval "$CONTAINER" # setup container (Singularity, e.g. "cmssw-el7")
+  peval "source setenv.sh" # set environment again (incl. functions) after Singularity
+  [ $VERB -ge 1 ] && { peval "env"; pbar; } # print out environment for debugging
+fi
 
-# ENVIRONMENT
-if [ ! -z "$CMSSW_BASE" -a -d "$CMSSW_BASE/src" ]; then
-  peval "cd $CMSSW_BASE/src"
-  peval 'eval `scramv1 runtime -sh`'
-  peval "cd $WORKDIR"
+# CMSSW ENVIRONMENT
+if [ -z "$CMSSW_BASE" ]; then # $CMSSW_BASE is not set
+  echo ">>> WARNING! CMSSW_BASE was not defined!"
+  # NOTE: HTCondor on lxplus does not seem to transfer environment as expected...
+  # Guess CMSSW_BASE path from $CMSSW_BASE/src/TauFW/PicoProducer/python/batch/submit_HTCondor.sh
+  SCRIPT="$(echo $TASKCMD | awk '{ print $2 }')" # assume `[COMMAND] [SCRIPT] [OPTIONS]`
+  CMSSW_BASE=$(realpath $(dirname "${SCRIPT}")/../../../../..)
+  echo ">>> Guessing CMSSW_BASE=$CMSSW_BASE based on SCRIPT=$SCRIPT"
+fi
+if [ ! -z "$CMSSW_BASE" ]; then # $CMSSW_BASE is set
+  echo ">>> Setting CMSSW environment from CMSSW_BASE=$CMSSW_BASE..."
+  if [ -d "/cvmfs/cms.cern.ch/" ]; then # /cvmfs exists/mounted
+    peval "source /cvmfs/cms.cern.ch/cmsset_default.sh"
+  else # could not find /cvmfs
+    echo ">>> WARNING! /cvmfs/cms.cern.ch/ does not exist or not mounted on machine $(uname -a), host $HOSTNAME !"
+  fi
+  if [ -d "$CMSSW_BASE/src" ]; then # $CMSSW_BASE exists/mounted
+    peval "cd $CMSSW_BASE/src"
+    peval 'eval $(scramv1 runtime -sh)' # = cmsenv
+    peval "cd $WORKDIR"
+  else # could not find CMSSW
+    echo ">>> WARNING! $CMSSW_BASE/src does not exist or not mounted on machine $(uname -a), host $HOSTNAME !"
+  fi
 fi
 
 # MAIN FUNCTIONALITY
-#TASKCMD=$(cat $JOBLIST | sed "${TASKID}q;d")
+pbar
+#TASKCMD=$(cat $JOBLIST | sed "${TASKID}q;d") # get TASKCMD from job list file
 echo "\$PWD=$PWD"
 peval "$TASKCMD"
 
 # FINISH
 echo
-END=`date +%s`; RUNTIME=$((END-START))
-echo "Job complete at `date`"
+pbar
+peval "rm $WORKDIR/setenv.sh"
+END="$(date +%s)"; RUNTIME=$((END-START))
+echo "Job complete at $(date)"
 printf "Took %d minutes %d seconds" "$(( $RUNTIME / 60 ))" "$(( $RUNTIME % 60 ))"
diff --git a/PicoProducer/python/batch/submit_HTCondor.sub b/PicoProducer/python/batch/submit_HTCondor.sub
index a2aa18aa1..fc2b1cce1 100644
--- a/PicoProducer/python/batch/submit_HTCondor.sub
+++ b/PicoProducer/python/batch/submit_HTCondor.sub
@@ -10,10 +10,17 @@ output                = $(mylogfile)
 error                 = $(mylogfile)
 should_transfer_files = no
 use_x509userproxy     = true
-getenv                = true
-environment           = JOBID=$(ClusterId);TASKID=$(ProcId)
+# NOTE: the following 'container' line is used to set the OS environment with a container/singularity,
+#       e.g. by adding the `-append container=cmssw-el7` option to condor_submit
+container             = ''
+# NOTE: `getenv = true` may put a large load on HTCondor infrastructure for the large CMSSW
+#       environment and many jobs, so it's better to set the environment during the job itself
+getenv                = CMSSW*, SCRAM_ARCH*
+# NOTE: Make sure to use quotation marks and spaces to avoid issues with the following line on lxplus:
+environment           = "JOBID=$(ClusterId) TASKID=$(ProcId) CONTAINER=$(container)"
+# NOTE: The following OS should be replaced for CMSSW 13 & later,
+#       or replaced with singularities all together when CentOS7 is phased out on lxplus
 requirements          = (OpSysAndVer =?= "CentOS7")
 +JobFlavour           = workday
 +MaxRuntime           = 20000
-#+AccountingGroup      = "group_u_BE.ABP.SLAP"
-#queue arg from args.txt
+#queue arg from args.txt
\ No newline at end of file
diff --git a/PicoProducer/python/batch/submit_HTCondor_CMG.sub b/PicoProducer/python/batch/submit_HTCondor_CMG.sub
index 7c912c42c..c2d30feab 100644
--- a/PicoProducer/python/batch/submit_HTCondor_CMG.sub
+++ b/PicoProducer/python/batch/submit_HTCondor_CMG.sub
@@ -1,5 +1,5 @@
 # Submit as
-#   condor_submit submit_HTCondor.sub 'mylogfile=log/myjob.$(ClusterId).$(ProcId).log' -queue arg from args.txt
+#   condor_submit submit_HTCondor_CMG.sub 'mylogfile=log/myjob.$(ClusterId).$(ProcId).log' -queue arg from args.txt
 universe              = vanilla
 executable            = python/batch/submit_HTCondor.sh
 arguments             = $(arg)
@@ -10,8 +10,17 @@ output                = $(mylogfile)
 error                 = $(mylogfile)
 should_transfer_files = no
 use_x509userproxy     = true
-getenv                = true
-environment           = JOBID=$(ClusterId);TASKID=$(ProcId)
+# NOTE: the following 'container' line is used to set the OS environment with a container/singularity,
+#       e.g. by adding the `-append container=cmssw-el7` option to condor_submit
+container             = ''
+# NOTE: `getenv = true` may put a large load on HTCondor infrastructure for the large CMSSW
+#       environment and many jobs, so it's better to set the environment during the job itself
+getenv                = CMSSW*, SCRAM_ARCH*
+# NOTE: Make sure to use quotation marks (") to avoid issues with the following line on lxplus:
+environment           = "JOBID=$(ClusterId) TASKID=$(ProcId) CONTAINER=$(container)"
+# NOTE: The following OS should be replaced for CMSSW 13 & later,
+#       or replaced with singularities all together when CentOS7 is phased out on lxplus
+requirements          = (OpSysAndVer =?= "CentOS7")
 +JobFlavour           = workday
 +MaxRuntime           = 20000
 +AccountingGroup      = "group_u_CMST3.all"

From c250a6b72e599245d92f3e35e6cd6b9f64e00db7 Mon Sep 17 00:00:00 2001
From: IzaakWN <iwn_@hotmail.com>
Date: Mon, 22 Apr 2024 19:29:26 +0200
Subject: [PATCH 04/10] fix lxplus HTCondor environment issue; add possibility
 for CMSSW singularity

---
 PicoProducer/python/batch/submit_HTCondor_CMG.sub | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/PicoProducer/python/batch/submit_HTCondor_CMG.sub b/PicoProducer/python/batch/submit_HTCondor_CMG.sub
index c2d30feab..46e1c848d 100644
--- a/PicoProducer/python/batch/submit_HTCondor_CMG.sub
+++ b/PicoProducer/python/batch/submit_HTCondor_CMG.sub
@@ -24,4 +24,4 @@ requirements          = (OpSysAndVer =?= "CentOS7")
 +JobFlavour           = workday
 +MaxRuntime           = 20000
 +AccountingGroup      = "group_u_CMST3.all"
-#queue arg from args.txt
+#queue arg from args.txt
\ No newline at end of file

From 0fee5990bbecac840a38f48ab2a51a5c2ed77ceb Mon Sep 17 00:00:00 2001
From: IzaakWN <iwn_@hotmail.com>
Date: Mon, 22 Apr 2024 20:02:49 +0200
Subject: [PATCH 05/10] debug EOS.cp

---
 PicoProducer/python/storage/EOS.py |  8 ++---
 PicoProducer/test/testStorage.py   | 54 +++++++++++++++++-------------
 2 files changed, 34 insertions(+), 28 deletions(-)

diff --git a/PicoProducer/python/storage/EOS.py b/PicoProducer/python/storage/EOS.py
index 78d4f6b8a..440d6e2c0 100644
--- a/PicoProducer/python/storage/EOS.py
+++ b/PicoProducer/python/storage/EOS.py
@@ -13,6 +13,7 @@ def __init__(self,path,verb=0,ensure=False,eos=False,**kwargs):
     """EOS is mounted on lxplus, so no special overload are necessary."""
     super(EOS,self).__init__(path,verb=verb,ensure=False,**kwargs)
     if not self.mounted: # EOS is mounted on lxplus
+      self.fileurl = "root://eosuser.cern.ch/" #"root://eoscms/"
       if eos: # use EOS command
         # https://cern.service-now.com/service-portal?id=kb_article&n=KB0001998
         os.environ["EOS_MGM_URL"] = "root://eosuser.cern.ch"
@@ -24,13 +25,12 @@ def __init__(self,path,verb=0,ensure=False,eos=False,**kwargs):
         self.lsurl = "root://eoscms.cern.ch/"
         self.lscol = -1 # take last column
         self.mkdrcmd = unset+"gfal-mkdir -p"
-        self.mkdrurl = "root://eosuser.cern.ch/"
+        self.mkdrurl = self.fileurl
         self.rmcmd = unset+"gfal-rm -r"
-        self.rmurl = "root://eosuser.cern.ch/"
+        self.rmurl = self.fileurl
       self.cpcmd   = 'xrdcp -f'
+      self.cpurl   = self.fileurl
       self.chmdprm = '2777'
-      self.cpurl   = "root://eoscms.cern.ch/"
-      self.fileurl = "root://eosuser.cern.ch/" #"root://eoscms/"
       #self.prefix  = "root://eoscms.cern.ch/"
     self.tmpdir    = '/tmp/$USER/'
     if ensure:
diff --git a/PicoProducer/test/testStorage.py b/PicoProducer/test/testStorage.py
index 790913919..2554ff40e 100755
--- a/PicoProducer/test/testStorage.py
+++ b/PicoProducer/test/testStorage.py
@@ -8,6 +8,7 @@
 from ROOT import gRandom, TFile, TTree, TH1F
 from array import array
 from datetime import datetime
+import traceback
 #from TauFW.common.tools.file import ensuremodule
 
 
@@ -99,28 +100,33 @@ def testStorage(path,readonly=False,hadd=True,verb=0):
     print(">>> Read only. Skip test for cp, rm, mkdir, hadd...")
     return
   
-  # CP
-  LOG.header("cp")
-  fname = createdummy("testStorage.txt")
-  LOG.color("storage.cp(%r,verb=%d)"%(fname,verb))
-  storage.cp(fname,verb=verb)
-  storage.ls(verb=verb)
-  
-  # EXISTS
-  LOG.header("exists")
-  LOG.color("storage.exists(%r,verb=%d)"%(fname,verb))
-  result = storage.exists(fname,verb=verb)
-  print(">>> Exists: %r"%(result))
-  storage.ls(verb=verb)
-  
-  # RM
-  LOG.header("rm")
-  LOG.color("storage.rm(%r,verb=%d)"%(fname,verb))
   try:
-    storage.rm(fname,verb=verb)
-  except Exception as error:
-    print(error)
-  storage.ls(verb=verb)
+    
+    # CP
+    LOG.header("cp")
+    fname = createdummy("testStorage.txt")
+    LOG.color("storage.cp(%r,verb=%d)"%(fname,verb))
+    storage.cp(fname,verb=verb)
+    storage.ls(verb=verb)
+    
+    # EXISTS
+    LOG.header("exists")
+    LOG.color("storage.exists(%r,verb=%d)"%(fname,verb))
+    result = storage.exists(fname,verb=verb)
+    print(">>> Exists: %r"%(result))
+    storage.ls(verb=verb)
+    
+    # RM
+    LOG.header("rm")
+    LOG.color("storage.rm(%r,verb=%d)"%(fname,verb))
+    try:
+      storage.rm(fname,verb=verb)
+    except Exception as error:
+      print(error)
+    storage.ls(verb=verb)
+  
+  except Exception:
+    print(traceback.format_exc())
   
   # MKDIR
   LOG.header("mkdir")
@@ -133,7 +139,7 @@ def testStorage(path,readonly=False,hadd=True,verb=0):
     result = storage.exists(dirname,verb=verb)
     print(">>> Exists: %r"%(result))
   except Exception as error:
-    print(error)
+    print(traceback.format_exc())
   
   # RM DIRECTORY
   LOG.header("rm directory")
@@ -144,7 +150,7 @@ def testStorage(path,readonly=False,hadd=True,verb=0):
       storage.rm(dirname,verb=verb)
       storage.ls(verb=verb)
     except Exception as error:
-      print(error)
+      print(traceback.format_exc())
   
   # HADD
   if hadd:
@@ -158,7 +164,7 @@ def testStorage(path,readonly=False,hadd=True,verb=0):
         storage.ls(verb=verb)
         storage.rm(outfile,verb=verb)
       except Exception as error:
-        print(error)
+        print(traceback.format_exc())
   
 
 def main(args):

From 438712983db6c2dc5e7f64b0d2b50c722acbb964 Mon Sep 17 00:00:00 2001
From: IzaakWN <iwn_@hotmail.com>
Date: Mon, 22 Apr 2024 20:09:51 +0200
Subject: [PATCH 06/10] debug EOS.ls URL

---
 PicoProducer/python/storage/EOS.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/PicoProducer/python/storage/EOS.py b/PicoProducer/python/storage/EOS.py
index 440d6e2c0..a56e4672b 100644
--- a/PicoProducer/python/storage/EOS.py
+++ b/PicoProducer/python/storage/EOS.py
@@ -22,7 +22,7 @@ def __init__(self,path,verb=0,ensure=False,eos=False,**kwargs):
       else: # NOTE: uberftp no longer supported for EOS...
         unset = "" if 'ucl' in host else "LD_LIBRARY_PATH='' PYTHONPATH='' " # unset libraries that break gFal tools
         self.lscmd = unset+"gfal-ls -l"
-        self.lsurl = "root://eoscms.cern.ch/"
+        self.lsurl =  self.fileurl #"root://eoscms.cern.ch/"
         self.lscol = -1 # take last column
         self.mkdrcmd = unset+"gfal-mkdir -p"
         self.mkdrurl = self.fileurl

From 8d347504085e4270a3955de814623ff1717e1564 Mon Sep 17 00:00:00 2001
From: IzaakWN <iwn_@hotmail.com>
Date: Tue, 23 Apr 2024 14:02:39 +0200
Subject: [PATCH 07/10] catch error for gFal tools; catch more exception and
 printout in color

---
 PicoProducer/python/storage/StorageSystem.py |   2 +-
 PicoProducer/test/testStorage.py             | 111 +++++++++++--------
 2 files changed, 68 insertions(+), 45 deletions(-)

diff --git a/PicoProducer/python/storage/StorageSystem.py b/PicoProducer/python/storage/StorageSystem.py
index 271da6cee..b559d1f76 100644
--- a/PicoProducer/python/storage/StorageSystem.py
+++ b/PicoProducer/python/storage/StorageSystem.py
@@ -126,7 +126,7 @@ def ls(self,*paths,**kwargs):
     retlist = [x for x in retlist.split(delim) if x]
     if isinstance(lscol,int):
       retlist = [l.split(' ')[lscol] for l in retlist]
-    if retlist and 'No such file or directory' in retlist[0]:
+    if retlist and any(e in retlist[0] for e in ['No such file or directory','[ERROR]']):
       LOG.warning(retlist[0])
       retlist = [ ]
     elif filters:
diff --git a/PicoProducer/test/testStorage.py b/PicoProducer/test/testStorage.py
index 790913919..7efe2ec08 100755
--- a/PicoProducer/test/testStorage.py
+++ b/PicoProducer/test/testStorage.py
@@ -8,6 +8,7 @@
 from ROOT import gRandom, TFile, TTree, TH1F
 from array import array
 from datetime import datetime
+import traceback
 #from TauFW.common.tools.file import ensuremodule
 
 
@@ -68,59 +69,78 @@ def testStorage(path,readonly=False,hadd=True,verb=0):
   for patharg in pathargs:
     for pathkwarg in pathkwargs:
       LOG.color("storage.expandpath(%s,%s)"%(','.join(repr(a) for a in patharg),','.join("%s=%r"%(k,v) for k,v in pathkwarg.items())))
-      result = storage.expandpath(*patharg,**pathkwarg)
-      print(">>>   %r"%(result))
+      try:
+        result = storage.expandpath(*patharg,**pathkwarg)
+        print(">>>   %r"%(result))
+      except:
+        LOG.error("storage.expandpath failed")
+        print(traceback.format_exc())
   
   # LS
   LOG.header("ls")
   LOG.color("storage.ls(verb=%d)"%(verb))
-  contents = storage.ls(verb=verb)
-  print(">>> Found %d items"%(len(contents)))
-  print(">>> Contents: %s"%(contents))
+  try:
+    contents = storage.ls(verb=verb)
+    print(">>> Found %d items"%(len(contents)))
+    print(">>> Contents: %s"%(contents))
+  except:
+    LOG.error("storage.ls failed")
+    print(traceback.format_exc())
   
   # FILES
   LOG.header("getfiles")
   LOG.color("storage.getfiles(verb=%d)"%(verb))
-  contents = storage.getfiles(verb=verb)
-  print(">>> Found %d items"%(len(contents)))
-  print(">>> Contents: %s"%(contents))
-  print(">>> ")
-  LOG.color("storage.getfiles(filter='*.*',verb=%d)"%(verb))
-  contents = storage.getfiles(filter='*.*',verb=verb)
-  print(">>> Found %d files"%(len(contents)))
-  print(">>> Contents: %s"%(contents))
-  print(">>> ")
-  LOG.color("storage.getfiles(filter='*.*',url=None,verb=%d)"%(verb))
-  contents = storage.getfiles(filter='*.*',url=None,verb=verb)
-  print(">>> Found %d files"%(len(contents)))
-  print(">>> Contents: %s"%(contents))
+  try:
+    contents = storage.getfiles(verb=verb)
+    print(">>> Found %d items"%(len(contents)))
+    print(">>> Contents: %s"%(contents))
+    print(">>> ")
+    LOG.color("storage.getfiles(filter='*.*',verb=%d)"%(verb))
+    contents = storage.getfiles(filter='*.*',verb=verb)
+    print(">>> Found %d files"%(len(contents)))
+    print(">>> Contents: %s"%(contents))
+    print(">>> ")
+    LOG.color("storage.getfiles(filter='*.*',url=None,verb=%d)"%(verb))
+    contents = storage.getfiles(filter='*.*',url=None,verb=verb)
+    print(">>> Found %d files"%(len(contents)))
+    print(">>> Contents: %s"%(contents))
+  except:
+    LOG.error("storage.getfiles failed")
+    print(traceback.format_exc())
   
   if readonly:
     print(">>> Read only. Skip test for cp, rm, mkdir, hadd...")
     return
   
-  # CP
-  LOG.header("cp")
-  fname = createdummy("testStorage.txt")
-  LOG.color("storage.cp(%r,verb=%d)"%(fname,verb))
-  storage.cp(fname,verb=verb)
-  storage.ls(verb=verb)
-  
-  # EXISTS
-  LOG.header("exists")
-  LOG.color("storage.exists(%r,verb=%d)"%(fname,verb))
-  result = storage.exists(fname,verb=verb)
-  print(">>> Exists: %r"%(result))
-  storage.ls(verb=verb)
-  
-  # RM
-  LOG.header("rm")
-  LOG.color("storage.rm(%r,verb=%d)"%(fname,verb))
   try:
-    storage.rm(fname,verb=verb)
-  except Exception as error:
-    print(error)
-  storage.ls(verb=verb)
+    
+    # CP
+    LOG.header("cp")
+    fname = createdummy("testStorage.txt")
+    LOG.color("storage.cp(%r,verb=%d)"%(fname,verb))
+    storage.cp(fname,verb=verb)
+    storage.ls(verb=verb)
+    
+    # EXISTS
+    LOG.header("exists")
+    LOG.color("storage.exists(%r,verb=%d)"%(fname,verb))
+    result = storage.exists(fname,verb=verb)
+    print(">>> Exists: %r"%(result))
+    storage.ls(verb=verb)
+    
+    # RM
+    LOG.header("rm")
+    LOG.color("storage.rm(%r,verb=%d)"%(fname,verb))
+    try:
+      storage.rm(fname,verb=verb)
+    except:
+      LOG.error("storage.rm failed")
+      print(traceback.format_exc())
+    storage.ls(verb=verb)
+  
+  except:
+    LOG.error("storage.cp/exists/ls file failed")
+    print(traceback.format_exc())
   
   # MKDIR
   LOG.header("mkdir")
@@ -133,7 +153,8 @@ def testStorage(path,readonly=False,hadd=True,verb=0):
     result = storage.exists(dirname,verb=verb)
     print(">>> Exists: %r"%(result))
   except Exception as error:
-    print(error)
+    LOG.error("storage.mkdir/ls/exists directory failed")
+    print(traceback.format_exc())
   
   # RM DIRECTORY
   LOG.header("rm directory")
@@ -143,8 +164,9 @@ def testStorage(path,readonly=False,hadd=True,verb=0):
     try:
       storage.rm(dirname,verb=verb)
       storage.ls(verb=verb)
-    except Exception as error:
-      print(error)
+    except:
+      LOG.error("storage.rm/ls directory failed")
+      print(traceback.format_exc())
   
   # HADD
   if hadd:
@@ -157,8 +179,9 @@ def testStorage(path,readonly=False,hadd=True,verb=0):
         storage.hadd(infiles,outfile,tmpdir=tmpdir,verb=verb)
         storage.ls(verb=verb)
         storage.rm(outfile,verb=verb)
-      except Exception as error:
-        print(error)
+      except:
+        LOG.error("storage.hadd/ls/rm ROOT file failed")
+        print(traceback.format_exc())
   
 
 def main(args):

From c90ae54c2a81e84b6705a5242f91e6c92d6526bc Mon Sep 17 00:00:00 2001
From: IzaakWN <iwn_@hotmail.com>
Date: Tue, 23 Apr 2024 14:00:39 +0200
Subject: [PATCH 08/10] take container/singularity from environment (usefull if
 already inside container)

---
 PicoProducer/python/batch/submit_HTCondor.sh  | 91 ++++++++++++++-----
 PicoProducer/python/batch/submit_HTCondor.sub | 18 +++-
 .../python/batch/submit_HTCondor_CMG.sub      | 20 +++-
 3 files changed, 96 insertions(+), 33 deletions(-)

diff --git a/PicoProducer/python/batch/submit_HTCondor.sh b/PicoProducer/python/batch/submit_HTCondor.sh
index 7073e4582..089fc4cf2 100755
--- a/PicoProducer/python/batch/submit_HTCondor.sh
+++ b/PicoProducer/python/batch/submit_HTCondor.sh
@@ -2,39 +2,80 @@
 ## Script to run on a HTCondor batch system
 
 # START
-START=`date +%s`
-echo "Job start at `date`"
-echo "Running job on machine `uname -a`, host $HOSTNAME"
-function peval { echo ">>> $@"; eval "$@"; }
+START="$(date +%s)"
+echo "Job starts at $(date)"
+echo "Running job on machine $(uname -a), host $HOSTNAME"
 
-# SETTING
-TASKCMD="$@"
-WORKDIR="$PWD"
-printf '=%.0s' `seq 60`; echo
-echo "\$PWD=$PWD"
-echo "\$JOBID=$JOBID"
-echo "\$TASKID=$TASKID"
-echo "\$HOSTNAME=$HOSTNAME"
-echo "\$TASKCMD=$TASKCMD"
-echo "\$WORKDIR=$WORKDIR"
-#printf '=%.0s' `seq 60`; echo
-#env
-#printf '=%.0s' `seq 60`; echo
+# SETTINGS: store in file for reuse
+cat << EOF > setenv.sh
+function peval { echo ">>> \$@"; eval "\$@"; }
+function pbar { printf '=%.0s' \$(seq \${1:-70}); echo; }
+VERB=${VERB:-0} # verbosity level for debugging
+START=$START
+JOBID=$JOBID
+TASKID=$TASKID
+HOSTNAME=$HOSTNAME
+PWD=$PWD
+WORKDIR=$PWD
+CONTAINER='${CONTAINER:-${APPTAINER_CONTAINER:-${SINGULARITY_CONTAINER:-}}}' # to set OS environment with container (Singularity, e.g. "cmssw-el7")
+CMSSW_BASE=$CMSSW_BASE # to set CMSSW environment
+TASKCMD='$@'
+EOF
+source setenv.sh # set environment
+pbar
+peval 'tail -n +3 setenv.sh | while read line; do echo "\$$line"; done'
+pbar
+[ $VERB -ge 1 ] && { peval "env"; pbar; } # print out environment for debugging
+
+# OS ENVIRONMENT with container (Singularity)
+# https://cms-sw.github.io/singularity.html
+# https://apptainer.org/docs/user/main/environment_and_metadata.html
+if [ ! -z "$CONTAINER" ]; then # if $CONTAINER is set
+  echo ">>> Setting OS environment with container '$CONTAINER'..."
+  if [[ "$CONTAINER" = *"/"* ]]; then # container/singularity image, e.g. "/cvmfs/unpacked.cern.ch/registry.hub.docker.com/cmssw/el7:x86_64"
+    peval "cmssw-env --cmsos \$(basename $CONTAINER)"
+  else # container/singularity command, e.g. just "cmssw-el7"
+    peval "$CONTAINER" # setup container
+  fi
+  peval "source setenv.sh" # set environment again (incl. functions) after Singularity
+  [ $VERB -ge 1 ] && { peval "env"; pbar; } # print out environment for debugging
+fi
 
-# ENVIRONMENT
-if [ ! -z "$CMSSW_BASE" -a -d "$CMSSW_BASE/src" ]; then
-  peval "cd $CMSSW_BASE/src"
-  peval 'eval `scramv1 runtime -sh`'
-  peval "cd $WORKDIR"
+# CMSSW ENVIRONMENT
+if [ -z "$CMSSW_BASE" ]; then # $CMSSW_BASE is not set
+  echo ">>> WARNING! CMSSW_BASE was not defined!"
+  # NOTE: HTCondor on lxplus does not seem to transfer environment as expected...
+  # Guess CMSSW_BASE path from $CMSSW_BASE/src/TauFW/PicoProducer/python/batch/submit_HTCondor.sh
+  SCRIPT="$(echo $TASKCMD | awk '{ print $2 }')" # assume `[COMMAND] [SCRIPT] [OPTIONS]`
+  CMSSW_BASE=$(realpath $(dirname "${SCRIPT}")/../../../../..)
+  echo ">>> Guessing CMSSW_BASE=$CMSSW_BASE based on SCRIPT=$SCRIPT"
+fi
+if [ ! -z "$CMSSW_BASE" ]; then # $CMSSW_BASE is set
+  echo ">>> Setting CMSSW environment from CMSSW_BASE=$CMSSW_BASE..."
+  if [ -d "/cvmfs/cms.cern.ch/" ]; then # /cvmfs exists/mounted
+    peval "source /cvmfs/cms.cern.ch/cmsset_default.sh"
+  else # could not find /cvmfs
+    echo ">>> WARNING! /cvmfs/cms.cern.ch/ does not exist or not mounted on machine $(uname -a), host $HOSTNAME !"
+  fi
+  if [ -d "$CMSSW_BASE/src" ]; then # $CMSSW_BASE exists/mounted
+    peval "cd $CMSSW_BASE/src"
+    peval 'eval $(scramv1 runtime -sh)' # = cmsenv
+    peval "cd $WORKDIR"
+  else # could not find CMSSW
+    echo ">>> WARNING! $CMSSW_BASE/src does not exist or not mounted on machine $(uname -a), host $HOSTNAME !"
+  fi
 fi
 
 # MAIN FUNCTIONALITY
-#TASKCMD=$(cat $JOBLIST | sed "${TASKID}q;d")
+pbar
+#TASKCMD=$(cat $JOBLIST | sed "${TASKID}q;d") # get TASKCMD from job list file
 echo "\$PWD=$PWD"
 peval "$TASKCMD"
 
 # FINISH
 echo
-END=`date +%s`; RUNTIME=$((END-START))
-echo "Job complete at `date`"
+pbar
+peval "rm $WORKDIR/setenv.sh"
+END="$(date +%s)"; RUNTIME=$((END-START))
+echo "Job complete at $(date)"
 printf "Took %d minutes %d seconds" "$(( $RUNTIME / 60 ))" "$(( $RUNTIME % 60 ))"
diff --git a/PicoProducer/python/batch/submit_HTCondor.sub b/PicoProducer/python/batch/submit_HTCondor.sub
index a2aa18aa1..40c20e3da 100644
--- a/PicoProducer/python/batch/submit_HTCondor.sub
+++ b/PicoProducer/python/batch/submit_HTCondor.sub
@@ -10,10 +10,20 @@ output                = $(mylogfile)
 error                 = $(mylogfile)
 should_transfer_files = no
 use_x509userproxy     = true
-getenv                = true
-environment           = JOBID=$(ClusterId);TASKID=$(ProcId)
+# NOTE:
+#   The following 'container' line is used to explicitly set the OS environment with a container/singularity,
+#   by adding the `-append container=cmssw-cc7` option to condor_submit, or defining `export CONTAINER=cmssw-cc7`
+#   in the shell's environment. Alternatively, if APPTAINER_CONTAINER or CONTAINER_CONTAINER is defined and
+#   exported in your environment (e.g. inside a container), the shell script use this with `cmssw-env`.
+container             = $ENV(CONTAINER:$ENV(APPTAINER_CONTAINER:$ENV(SINGULARITY_CONTAINER:)))
+# NOTE: `getenv = true` may put a large load on HTCondor infrastructure for the large CMSSW
+#       environment and many jobs, so it's better to set the environment during the job itself
+getenv                = CMSSW*, SCRAM_ARCH*, *CONTAINER
+# NOTE: Make sure to use quotation marks (") to avoid issues with the following line on lxplus:
+environment           = "JOBID=$(ClusterId) TASKID=$(ProcId) CONTAINER=$(container)"
+# NOTE: The following OS requirement should be replaced for CMSSW 13 & later,
+#       or replaced with singularities all together when CentOS7 is phased out on lxplus
 requirements          = (OpSysAndVer =?= "CentOS7")
 +JobFlavour           = workday
 +MaxRuntime           = 20000
-#+AccountingGroup      = "group_u_BE.ABP.SLAP"
-#queue arg from args.txt
+#queue arg from args.txt
\ No newline at end of file
diff --git a/PicoProducer/python/batch/submit_HTCondor_CMG.sub b/PicoProducer/python/batch/submit_HTCondor_CMG.sub
index 7c912c42c..5c35fa898 100644
--- a/PicoProducer/python/batch/submit_HTCondor_CMG.sub
+++ b/PicoProducer/python/batch/submit_HTCondor_CMG.sub
@@ -1,5 +1,5 @@
 # Submit as
-#   condor_submit submit_HTCondor.sub 'mylogfile=log/myjob.$(ClusterId).$(ProcId).log' -queue arg from args.txt
+#   condor_submit submit_HTCondor_CMG.sub 'mylogfile=log/myjob.$(ClusterId).$(ProcId).log' -queue arg from args.txt
 universe              = vanilla
 executable            = python/batch/submit_HTCondor.sh
 arguments             = $(arg)
@@ -10,9 +10,21 @@ output                = $(mylogfile)
 error                 = $(mylogfile)
 should_transfer_files = no
 use_x509userproxy     = true
-getenv                = true
-environment           = JOBID=$(ClusterId);TASKID=$(ProcId)
+# NOTE:
+#   The following 'container' line is used to explicitly set the OS environment with a container/singularity,
+#   by adding the `-append container=cmssw-cc7` option to condor_submit, or defining `export CONTAINER=cmssw-cc7`
+#   in the shell's environment. Alternatively, if APPTAINER_CONTAINER or CONTAINER_CONTAINER is defined and
+#   exported in your environment (e.g. inside a container), the shell script use this with `cmssw-env`.
+container             = $ENV(CONTAINER:$ENV(APPTAINER_CONTAINER:$ENV(SINGULARITY_CONTAINER:)))
+# NOTE: `getenv = true` may put a large load on HTCondor infrastructure for the large CMSSW
+#       environment and many jobs, so it's better to set the environment during the job itself
+getenv                = CMSSW*, SCRAM_ARCH*, *CONTAINER
+# NOTE: Make sure to use quotation marks (") to avoid issues with the following line on lxplus:
+environment           = "JOBID=$(ClusterId) TASKID=$(ProcId) CONTAINER=$(container)"
+# NOTE: The following OS requirement should be replaced for CMSSW 13 & later,
+#       or replaced with singularities all together when CentOS7 is phased out on lxplus
+requirements          = (OpSysAndVer =?= "CentOS7")
 +JobFlavour           = workday
 +MaxRuntime           = 20000
 +AccountingGroup      = "group_u_CMST3.all"
-#queue arg from args.txt
+#queue arg from args.txt
\ No newline at end of file

From 08ac845a19e272ef9d3ddd50066a5b86a2ab3ea3 Mon Sep 17 00:00:00 2001
From: IzaakWN <iwn_@hotmail.com>
Date: Tue, 23 Apr 2024 14:43:52 +0200
Subject: [PATCH 09/10] expand environment variable

---
 PicoProducer/python/batch/submit_HTCondor.sh |  1 -
 PicoProducer/python/storage/EOS.py           |  1 -
 PicoProducer/python/storage/StorageSystem.py |  2 +-
 PicoProducer/python/storage/T2_PSI.py        |  2 +-
 PicoProducer/python/storage/T3_PSI.py        |  3 ++-
 common/python/tools/file.py                  | 18 ++++++++++--------
 6 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/PicoProducer/python/batch/submit_HTCondor.sh b/PicoProducer/python/batch/submit_HTCondor.sh
index 089fc4cf2..e0a3107f6 100755
--- a/PicoProducer/python/batch/submit_HTCondor.sh
+++ b/PicoProducer/python/batch/submit_HTCondor.sh
@@ -44,7 +44,6 @@ fi
 # CMSSW ENVIRONMENT
 if [ -z "$CMSSW_BASE" ]; then # $CMSSW_BASE is not set
   echo ">>> WARNING! CMSSW_BASE was not defined!"
-  # NOTE: HTCondor on lxplus does not seem to transfer environment as expected...
   # Guess CMSSW_BASE path from $CMSSW_BASE/src/TauFW/PicoProducer/python/batch/submit_HTCondor.sh
   SCRIPT="$(echo $TASKCMD | awk '{ print $2 }')" # assume `[COMMAND] [SCRIPT] [OPTIONS]`
   CMSSW_BASE=$(realpath $(dirname "${SCRIPT}")/../../../../..)
diff --git a/PicoProducer/python/storage/EOS.py b/PicoProducer/python/storage/EOS.py
index a56e4672b..a5b6f1bc7 100644
--- a/PicoProducer/python/storage/EOS.py
+++ b/PicoProducer/python/storage/EOS.py
@@ -32,7 +32,6 @@ def __init__(self,path,verb=0,ensure=False,eos=False,**kwargs):
       self.cpurl   = self.fileurl
       self.chmdprm = '2777'
       #self.prefix  = "root://eoscms.cern.ch/"
-    self.tmpdir    = '/tmp/$USER/'
     if ensure:
       self.ensuredir(self.path)
     
\ No newline at end of file
diff --git a/PicoProducer/python/storage/StorageSystem.py b/PicoProducer/python/storage/StorageSystem.py
index b559d1f76..93735ad99 100644
--- a/PicoProducer/python/storage/StorageSystem.py
+++ b/PicoProducer/python/storage/StorageSystem.py
@@ -29,7 +29,7 @@ def __init__(self,path,verb=0,**kwargs):
     self.chmdcmd = 'chmod'
     self.chmdurl = ''
     self.haddcmd = kwargs.get('haddcmd',None) or 'hadd -ff' # same compression level as first in put
-    self.tmpdir  = '/tmp/$USER/' # $TMPDIR # mounted temporary directory
+    self.tmpdir  = os.environ.get('TMPDIR',os.path.join('/tmp/',os.environ.get('USER','TauFW')) # mounted temporary directory
     self.fileurl = ""
     self.verbosity = verb
     if path.startswith('/'):
diff --git a/PicoProducer/python/storage/T2_PSI.py b/PicoProducer/python/storage/T2_PSI.py
index f1cda2471..b34c4daa2 100644
--- a/PicoProducer/python/storage/T2_PSI.py
+++ b/PicoProducer/python/storage/T2_PSI.py
@@ -19,7 +19,7 @@ def __init__(self,path,verb=0,ensure=False,**kwargs):
     #self.cpurl   = "root://storage01.lcg.cscs.ch/"
     self.cpcmd   = "LD_LIBRARY_PATH='' PYTHONPATH='' gfal-copy --force"
     self.cpurl   = "gsiftp://storage01.lcg.cscs.ch/"
-    self.tmpdir  = '/scratch/$USER/'
+    self.tmpdir  = os.path('/scratch/',os.environ.get('USER','TauFW'))
     self.fileurl = "root://storage01.lcg.cscs.ch/"
     if ensure:
       self.ensuredir(self.path)
diff --git a/PicoProducer/python/storage/T3_PSI.py b/PicoProducer/python/storage/T3_PSI.py
index 4f1fc3201..15c1974c1 100644
--- a/PicoProducer/python/storage/T3_PSI.py
+++ b/PicoProducer/python/storage/T3_PSI.py
@@ -1,5 +1,6 @@
 #! /usr/bin/env python
 # Author: Izaak Neutelings (May 2020)
+import os
 from TauFW.PicoProducer.storage.StorageSystem import StorageSystem
 
 
@@ -18,7 +19,7 @@ def __init__(self,path,verb=0,ensure=False,**kwargs):
       self.cpcmd   = 'xrdcp -f'
       self.cpurl   = "root://t3dcachedb03.psi.ch/"
     self.fileurl = "root://t3dcachedb03.psi.ch/"
-    self.tmpdir  = '/scratch/$USER/'
+    self.tmpdir  = os.path('/scratch/',os.environ.get('USER','TauFW'))
     if ensure:
       self.ensuredir(self.path)
     
diff --git a/common/python/tools/file.py b/common/python/tools/file.py
index 84fbe4347..ed260bc30 100644
--- a/common/python/tools/file.py
+++ b/common/python/tools/file.py
@@ -31,18 +31,20 @@ def ensuredir(*dirnames,**kwargs):
   """Make directory if it does not exist.
   If more than one path is given, it is joined into one."""
   dirname   = os.path.join(*dirnames)
-  empty     = kwargs.get('empty', False)
-  verbosity = kwargs.get('verb',  0    )
+  empty     = kwargs.get('empty', False )
+  verbosity = kwargs.get('verb',  0     )
   if not dirname:
-    pass
-  elif not os.path.exists(dirname):
-    os.makedirs(dirname)
+    return dirname
+  elif '$' in dirname: # expand environmental variables
+    dirname = os.path.expandvars(dirname)
+  if not os.path.exists(dirname):
     if verbosity>=1:
-      print(">>> Made directory %r"%(dirname))
+      print(">>> Making directory %r..."%(dirname))
+    os.makedirs(dirname)
     if not os.path.exists(dirname):
       print(">>> Failed to make directory %r"%(dirname))
-  elif empty:
-    for filename in os.listdir(dirname):
+  elif empty: # make sure directory is empty
+    for filename in os.listdir(dirname): # remove all contents
       filepath = os.path.join(dirname,filename)
       if os.path.isfile(filepath) or os.path.islink(filepath):
         os.unlink(filepath)

From 8a3b9848ee6a25267d1a0c61885a3cd7038777a1 Mon Sep 17 00:00:00 2001
From: IzaakWN <iwn_@hotmail.com>
Date: Tue, 23 Apr 2024 14:47:45 +0200
Subject: [PATCH 10/10] debug: close parentheses

---
 PicoProducer/python/storage/StorageSystem.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/PicoProducer/python/storage/StorageSystem.py b/PicoProducer/python/storage/StorageSystem.py
index 93735ad99..4745cea38 100644
--- a/PicoProducer/python/storage/StorageSystem.py
+++ b/PicoProducer/python/storage/StorageSystem.py
@@ -29,7 +29,7 @@ def __init__(self,path,verb=0,**kwargs):
     self.chmdcmd = 'chmod'
     self.chmdurl = ''
     self.haddcmd = kwargs.get('haddcmd',None) or 'hadd -ff' # same compression level as first in put
-    self.tmpdir  = os.environ.get('TMPDIR',os.path.join('/tmp/',os.environ.get('USER','TauFW')) # mounted temporary directory
+    self.tmpdir  = os.environ.get('TMPDIR',os.path.join('/tmp/',os.environ.get('USER','TauFW'))) # mounted temporary directory
     self.fileurl = ""
     self.verbosity = verb
     if path.startswith('/'):