Lint shell scripts

.github/workflows/check_shell_script.yml

@@ -0,0 +1,16 @@
+name: Check Shell Script Syntax
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/[email protected]
+    - name: Install ShellCheck
+      run: |
+        sudo apt-get install shellcheck
+    - name: Lint with ShellCheck
+      run: |
+        shellcheck --shell=bash -x $(find ./ -iname "*.sh")

CondorMonitoring/JobCounter.sh

@@ -8,10 +8,12 @@
 location="/afs/cern.ch/user/c/cmst1/CondorMonitoring"
 outputdir="/afs/cern.ch/user/c/cmst1/www/CondorMonitoring/"
 
-#Set environment 
+# Set environment 
+# shellcheck disable=SC1091
 source /data/admin/wmagent/env.sh
+# shellcheck disable=SC1091
 source /data/srv/wmagent/current/apps/wmagent/etc/profile.d/init.sh
-cd $location
+cd $location||exit
 
 #Email if things are running slowly
 if [ -f scriptRunning.run ];
@@ -33,6 +35,6 @@ fi
 python JobCounter.py &> JobCounter.log
 exitstatus="$?"
 echo "JobCounter.py exit status: $exitstatus"
-cp *.json $outputdir
-cp *.txt $outputdir
+cp ./*.json $outputdir
+cp ./*.txt $outputdir
 rm scriptRunning.run

PhEDEXFix/phedex-fix.sh

@@ -1,10 +1,12 @@
 #!/bin/bash
 ./config/wmagent/manage execute-agent wmcoreD --shutdown --components=PhEDExInjector
 ./config/wmagent/manage mysql-prompt wmagent<querry.sql>lfns.txt
+# shellcheck disable=SC2002
 cat lfns.txt | grep /store > lfnsgreped.txt
 #tail -n 28 install/wmagent/PhEDExInjector/ComponentLog | head -n1 > phedex.log
 #for j in $(for i in $(cat phedex.log) ; do echo $i | awk -F'"' {'print $2'} ; done | grep root) ; do echo $j; done > lfns.txt
 ./fileInPhedex.py lfnsgreped.txt > lfnsinPhedex.txt
+# shellcheck disable=SC2013
 for i in $(cat lfnsinPhedex.txt); do echo "UPDATE dbsbuffer_file set in_phedex = 1 where lfn='$i';";done > meuscript.sql
 ./config/wmagent/manage mysql-prompt wmagent < meuscript.sql
 ./config/wmagent/manage execute-agent wmcoreD --shutdown --components=PhEDExInjector

StoreResults/StoreResultsAddCMSSWReleases.sh

@@ -1,3 +1,4 @@
 #!/bin/bash
+# shellcheck disable=SC1091
 source /data/srv/wmagent/current/apps/wmagent/etc/profile.d/init.sh
 python2.6 /home/cmsdataops/storeResults/cronjobs/StoreResultsAddCMSSWReleases.py

Unified/exec_expose.sh

@@ -5,7 +5,8 @@ loc=$4
 dest=$5
 short=$6
 
-echo "Parameters passed to execution:" $wf $p $er $loc $dest $short
+echo "Parameters passed to execution:" "$wf" "$p" "$er" "$loc" "$dest" "$short"
 
 #sudo -u cmst1 /bin/bash --init-file ~cmst1/.bashrc $loc/WmAgentScripts/Unified/expose.sh $wf $p $er $dest $short
-source $loc/WmAgentScripts/Unified/expose.sh $wf $p $er $dest $short
+# shellcheck disable=SC2034,SC1090
+source "$loc"/WmAgentScripts/Unified/expose.sh "$wf" "$p" "$er" "$dest" "$short"

Unified/expose.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
-host=`echo $HOSTNAME | cut -d "." -f 1`
-echo "looking for condor log on" $host
+host=$(echo "$HOSTNAME" | cut -d "." -f 1)
+echo "looking for condor log on" "$host"
 taskName=$1
 wmbsID=$2
 code=$3
@@ -11,54 +11,55 @@ short=$5
 
 #end_dir=/afs/cern.ch/user/c/cmst1/www/JobLogs/$taskName/$cluster/$host\_$wmbsID
 #end_dir=/afs/cern.ch/user/c/cmst2/www/unified/condorlogs/$taskName/$code/$cluster/$host\_$wmbsID
+# shellcheck disable=SC1001
 end_dir=$loc/condorlogs/$taskName/$code/$short/$cluster/$host\_$wmbsID
 
 echo '==================================== Condor log retrieval ============================================='
-echo 'LISTING: ' /data/srv/wmagent/current/install/wmagent/JobCreator/JobCache/$taskName/*/job_$wmbsID/
-dir=`ls -d /data/srv/wmagent/current/install/wmagent/JobCreator/JobCache/$taskName/*/job_$wmbsID/`
-if [ -z $dir ]; then
+echo 'LISTING: ' /data/srv/wmagent/current/install/wmagent/JobCreator/JobCache/"$taskName"/*/job_"$wmbsID"/
+dir=$(ls -d /data/srv/wmagent/current/install/wmagent/JobCreator/JobCache/"$taskName"/*/job_"$wmbsID"/)
+if [ -z "$dir" ]; then
     echo "no on-going directory"
 else
-    echo "     Found some logs in" $dir
-    echo "Will put it in " $end_dir    
-    mkdir -p $end_dir
-    cp -r $dir/* $end_dir/.
+    echo "     Found some logs in $dir"
+    echo "Will put it in $end_dir"
+    mkdir -p "$end_dir"
+    cp -r "$dir"/* "$end_dir"/.
 fi
 echo '==================================== Condor log retrieval ============================================='
-echo 'LISTING: ' /data/srv/wmagent/current/install/wmagent/JobCreator/JobCache/$taskName/*/*/job_$wmbsID/
-dir=`ls -d /data/srv/wmagent/current/install/wmagent/JobCreator/JobCache/$taskName/*/*/job_$wmbsID/`
-if [ -z $dir ]; then
+echo 'LISTING: ' /data/srv/wmagent/current/install/wmagent/JobCreator/JobCache/"$taskName"/*/*/job_"$wmbsID"/
+dir=$(ls -d /data/srv/wmagent/current/install/wmagent/JobCreator/JobCache/"$taskName"/*/*/job_"$wmbsID"/)
+if [ -z "$dir" ]; then
     echo "no on-going directory"
 else
-    echo "      Found some logs in" $dir
-    echo "Will put it in " $end_dir    
-    mkdir -p $end_dir
-    cp -r $dir/* $end_dir/.
+    echo "      Found some logs in" "$dir"
+    echo "Will put it in " "$end_dir"    
+    mkdir -p "$end_dir"
+    cp -r "$dir"/* "$end_dir"/.
 fi
 
 echo '==================================== Condor log retrieval ============================================='
-echo 'LISTING: ' /data/srv/wmagent/current/install/wmagent/JobArchiver/logDir/${taskName:0:1}/$taskName/JobCluster_$cluster/Job_$wmbsID.tar.bz2
-file=`ls /data/srv/wmagent/current/install/wmagent/JobArchiver/logDir/${taskName:0:1}/$taskName/JobCluster_$cluster/Job_$wmbsID.tar.bz2`
-if [ -z $file ]; then
+echo 'LISTING: ' /data/srv/wmagent/current/install/wmagent/JobArchiver/logDir/"${taskName:0:1}"/"$taskName"/JobCluster_"$cluster"/Job_"$wmbsID".tar.bz2
+file=$(ls /data/srv/wmagent/current/install/wmagent/JobArchiver/logDir/"${taskName:0:1}"/"$taskName"/JobCluster_"$cluster"/Job_"$wmbsID".tar.bz2)
+if [ -z "$file" ]; then
     echo "no archived file"
 else
-    echo "      Found some logs in" $file
-    echo "Will put it in " $end_dir    
-    mkdir -p  $end_dir
-    cd $end_dir
-    tar xvf $file
+    echo "      Found some logs in" "$file"
+    echo "Will put it in " "$end_dir"    
+    mkdir -p  "$end_dir"
+    cd "$end_dir"||exit
+    tar xvf "$file"
 fi
 
 echo '==================================== Condor log retrieval ============================================='
 cluster=${wmbsID:0:4}
-echo 'LISTING: ' /data/srv/wmagent/current/install/wmagent/JobArchiver/logDir/${taskName:0:1}/$taskName/JobCluster_$cluster/Job_$wmbsID.tar.bz2
-file=`ls /data/srv/wmagent/current/install/wmagent/JobArchiver/logDir/${taskName:0:1}/$taskName/JobCluster_$cluster/Job_$wmbsID.tar.bz2`
-if [ -z $file ]; then
+echo 'LISTING: ' /data/srv/wmagent/current/install/wmagent/JobArchiver/logDir/"${taskName:0:1}"/"$taskName"/JobCluster_"$cluster"/Job_"$wmbsID".tar.bz2
+file=$(ls /data/srv/wmagent/current/install/wmagent/JobArchiver/logDir/"${taskName:0:1}"/"$taskName"/JobCluster_"$cluster"/Job_"$wmbsID".tar.bz2)
+if [ -z "$file" ]; then
     echo "no archived file"
 else
-    echo "      Found some logs in" $file
-    echo "Will put it in " $end_dir    
-    mkdir -p  $end_dir
-    cd $end_dir
-    tar xvf $file
+    echo "      Found some logs in" "$file"
+    echo "Will put it in " "$end_dir"    
+    mkdir -p  "$end_dir"
+    cd "$end_dir"||exit
+    tar xvf "$file"
 fi

WFComplete_submitter.sh

@@ -1,4 +1,4 @@
 #!/bin/sh
 
 echo "Executing $1 as cmst1..."
-sudo -u cmst1 /bin/bashs --init-file $1
+sudo -u cmst1 /bin/bashs --init-file "$1"

actcycle.sh

@@ -1,14 +1,18 @@
 BASE_DIR=/data/unified/WmAgentScripts/
+# shellcheck disable=SC2034
 HTML_DIR=/var/www/html/unified/
 
-lock_name=`echo $BASH_SOURCE | cut -f 1 -d "."`.lock
-source $BASE_DIR/cycle_common.sh $lock_name
+# shellcheck disable=SC1090,SC2128
+lock_name=$(echo "$BASH_SOURCE" | cut -f 1 -d ".").lock
+# shellcheck disable=SC1090
+source $BASE_DIR/cycle_common.sh "$lock_name"
 
 ## get sso cookie and new grid proxy
+# shellcheck disable=SC1090
 source $BASE_DIR/credentials.sh
 
 ## submit ACDCs and clones from actions submitted via new recovery tools
 $BASE_DIR/cWrap.sh Unified/actor.py
 
-rm -f $lock_name
+rm -f "$lock_name"
 

assigncycle.sh

@@ -1,8 +1,11 @@
 BASE_DIR=/data/unified/WmAgentScripts/
+# shellcheck disable=SC2034
 HTML_DIR=/var/www/html/unified/
 
-lock_name=`echo $BASH_SOURCE | cut -f 1 -d "."`.lock
-source $BASE_DIR/cycle_common.sh $lock_name
+# shellcheck disable=SC2128
+lock_name=$(echo "$BASH_SOURCE" | cut -f 1 -d ".").lock
+# shellcheck disable=SC1090
+source $BASE_DIR/cycle_common.sh "$lock_name"
 
 ## get the workflow in/out of the system
 $BASE_DIR/cWrap.sh Unified/injector.py
@@ -26,5 +29,5 @@ $BASE_DIR/cWrap.sh Unified/assignor.py _PR_ref
 ## assign the workflow to sites
 $BASE_DIR/cWrap.sh Unified/assignor.py
 
-rm -f $lock_name
+rm -f "$lock_name"
 

cWrap.sh

@@ -7,74 +7,78 @@ if [ ! -d $FINAL_HTML_DIR ] ; then
     echo "Cannot read the log destination",$FINAL_HTML_DIR
     exit
 fi
-cd $BASE_DIR
+cd $BASE_DIR||exit
 
-modulename=`echo $1 | sed 's/\.py//' | sed 's/Unified\///'`
-mkdir -p $HTML_DIR/logs/$modulename/
-mkdir -p $FINAL_HTML_DIR/logs/$modulename/
+modulename=$(echo "$1" | sed 's/\.py//' | sed 's/Unified\///')
+mkdir -p $HTML_DIR/logs/"$modulename"/
+mkdir -p $FINAL_HTML_DIR/logs/"$modulename"/
 
 last_log=$HTML_DIR/logs/$modulename/last.log
-s_dated_log=$modulename/`date +%F_%T`.log
+s_dated_log=$modulename/$(date +%F_%T).log
 dated_log=$HTML_DIR/logs/$s_dated_log
 log=$dated_log
 
-echo `date` > $log
-echo $$ >> $log
+# shellcheck disable=SC2005
+echo "$(date)" > "$log"
+echo $$ >> "$log"
 
 if [ -r unified_drain ] ; then
-    echo "System is locally draining" >> $log
-    cp $log $last_log
-    cp $log $FINAL_HTML_DIR/logs/$modulename/.
-    cp $log $FINAL_HTML_DIR/logs/$modulename/last.log
+    echo "System is locally draining" >> "$log"
+    cp "$log" "$last_log"
+    cp "$log" $FINAL_HTML_DIR/logs/"$modulename"/.
+    cp "$log" $FINAL_HTML_DIR/logs/"$modulename"/last.log
     exit
 fi
 if [ -r /eos/cms/store/unified/unified_drain ] ; then
-    echo "System is globally draining" >> $log
-    cp $log $last_log
-    cp $log $FINAL_HTML_DIR/logs/$modulename/.
-    cp $log $FINAL_HTML_DIR/logs/$modulename/last.log
+    echo "System is globally draining" >> "$log"
+    cp "$log" "$last_log"
+    cp "$log" $FINAL_HTML_DIR/logs/"$modulename"/.
+    cp "$log" $FINAL_HTML_DIR/logs/"$modulename"/last.log
     exit
 fi
-
-
-echo $USER >> $log
-echo $HOSTNAME >> $log
-echo module $modulename>> $log 
+# shellcheck disable=SC2129
+echo "$USER" >> "$log"
+echo "$HOSTNAME" >> "$log"
+echo module "$modulename">> "$log" 
 
 source ./set.sh
 
-echo >> $log
+echo >> "$log"
 
-start=`date +%s`
-python ssi.py $modulename $start
+start=$(date +%s)
+python ssi.py "$modulename" "$start"
 
-python $* &>> $log
+python "$@" &>> "$log"
 
+# shellcheck disable=SC2181
 if [ $? == 0 ]; then
-    echo "finished" >> $log
+    echo "finished" >> "$log"
 else
-    echo -e "\nAbnormal termination with exit code $?" >> $log
-    top -n1  -o %MEM -c >> $log
+    echo -e "\nAbnormal termination with exit code $?" >> "$log"
+    top -n1  -o %MEM -c >> "$log"
 
     emaillog=$log.txt
     failed_pid=$!
-    echo "Abnormal termination, check $log" > $emaillog
-    echo "https://cms-unified.web.cern.ch/cms-unified/logs/$s_dated_log" >> $emaillog
-    echo $failed_pid >> $emaillog
-    echo $USER >> $emaillog
-    echo $HOSTNAME >> $emaillog
-    echo -e "module $modulename \n" >> $emaillog 
-    tail $log >> $emaillog
-    cat $emaillog | mail -s "[Ops] module "$modulename" failed" [email protected]
+    echo "Abnormal termination, check $log" > "$emaillog"
+    # shellcheck disable=SC2129
+    echo "https://cms-unified.web.cern.ch/cms-unified/logs/$s_dated_log" >> "$emaillog"
+    echo $failed_pid >> "$emaillog"
+    echo "$USER" >> "$emaillog"
+    echo "$HOSTNAME" >> "$emaillog"
+    echo -e "module $modulename \n" >> "$emaillog" 
+    tail "$log" >> "$emaillog"
+    # shellcheck disable=SC2002
+    cat "$emaillog" | mail -s "[Ops] module $modulename failed" [email protected]
 fi
 
-stop=`date +%s`
-python ssi.py $modulename $start $stop
-echo `date` >> $log
+stop=$(date +%s)
+python ssi.py "$modulename" "$start" "$stop"
+# shellcheck disable=SC2005
+echo "$(date)" >> "$log"
 
-#cp $log $dated_log
-cp $log $last_log
-cp $log $FINAL_HTML_DIR/logs/$modulename/.
-cp $log $FINAL_HTML_DIR/logs/$modulename/last.log
+# cp $log $dated_log
+cp "$log" "$last_log"
+cp "$log" $FINAL_HTML_DIR/logs/"$modulename"/.
+cp "$log" $FINAL_HTML_DIR/logs/"$modulename"/last.log
 
-#rm $log
+# rm $log

checkmem.sh

@@ -1,8 +1,8 @@
 #log the memory of all
 for each in 0268 0269 0272 0273 0274 0275; do
-    lf='/data/unified/www/logs/checkmem/'`date +%F_%T`.$each.memlog
-    date > $lf
-    ssh vocms$each "ps -o pid,user,rss,%mem,args --sort -pmem -e | head -10" >> $lf
+    lf='/data/unified/www/logs/checkmem/'$(date +%F_%T).$each.memlog
+    date > "$lf"
+    ssh vocms$each "ps -o pid,user,rss,%mem,args --sort -pmem -e | head -10" >> "$lf"
 done
 ## keep it clean
 find /data/unified/www/logs/checkmem/ -type f -name '*.memlog' -mtime +1 -exec rm {} \;

cleanBackfills.sh

@@ -1,7 +1,9 @@
-cd /data/unified/WmAgentScripts
+# shellcheck disable=SC1091
+cd /data/unified/WmAgentScripts||exit
 
-oweek=`date +%W`
+oweek=$(date +%W)
 week=${oweek#0}
+# shellcheck disable=SC2219,SC2034
 let oddity=week%2
 
 export X509_USER_PROXY=$HOME/private/personal/voms_proxy.cert

clearcycle.sh

@@ -1,8 +1,11 @@
 BASE_DIR=/data/unified/WmAgentScripts/
+# shellcheck disable=SC2034
 HTML_DIR=/var/www/html/unified/
 
-lock_name=`echo $BASH_SOURCE | cut -f 1 -d "."`.lock
-source $BASE_DIR/cycle_common.sh $lock_name
+# shellcheck disable=SC2128
+lock_name=$(echo "$BASH_SOURCE" | cut -f 1 -d ".").lock
+# shellcheck disable=SC1090
+source $BASE_DIR/cycle_common.sh "$lock_name"
 
 $BASE_DIR/cWrap.sh Unified/completor.py
 
@@ -11,5 +14,5 @@ $BASE_DIR/cWrap.sh Unified/closor.py
 ## early announce what can be announced already
 $BASE_DIR/cWrap.sh Unified/closor.py --announce
 
-rm -f $lock_name
+rm -f "$lock_name"