shell: reformatted, fixed inspections, typos (#2506)

Reformatted shell scripts according to [ShellCheck](https://github.com/koalaman/shellcheck/).

I. Most common changes:
1. https://github.com/koalaman/shellcheck/wiki/SC2086
	`$var` → `"$var"`
	Note: this isn't always necessary and I've been careful not to substitute where it wasn't necessary in meaning.
2. https://github.com/koalaman/shellcheck/wiki/SC2006
	`` `command` `` → `$(command)`
3. https://github.com/koalaman/shellcheck/wiki/SC2004
	`$(( $a + $b ))` → `$(( a + b ))`
4. https://github.com/koalaman/shellcheck/wiki/SC2164
	`cd "$dir"` → `cd "$dir" || exit 1`
5. https://github.com/koalaman/shellcheck/wiki/SC2166
	`[ check1 -o check2 ]` → `[ check1 ] || [ check2 ]`
6. https://github.com/koalaman/shellcheck/wiki/SC2002
	`cat "${file}" | wc -c` → `< "${file}" wc -c`
	Note: this looks a bit uglier but works faster.

II. Some special changes:
1. In file `utils/common.sh`:
	https://github.com/koalaman/shellcheck/wiki/SC2112
	This script is interpreted by `sh`, not by `bash`, but uses the keyword `function`.
	So I replaced `#!/usr/bin/env sh` to `#!/usr/bin/env bash`.
2. After that I thought of replacing all shebangs to `#!/usr/bin/env bash` for consistency and cross-platform compatibility, especially since most of the files already use bash.
3. But in cases when it was `#!/bin/sh -e` or `#!/bin/bash -eu` another problem appears:
	https://github.com/koalaman/shellcheck/wiki/SC2096
	So I decided to make all shebangs look uniform:
	```
	#!/usr/bin/env bash
	set -e (or set -eu) (if needed)
	```
4. In file `tests/ossfuzz.sh`:
	https://github.com/koalaman/shellcheck/wiki/SC2162
	`read i` → `read -r i`
	Note: I think that there is no need in special treatment for backslashes, but I could be wrong.
5. In file `tests/do.sh.in`:
	https://github.com/koalaman/shellcheck/wiki/SC2035
	`ls *.*cap*` → `ls -- *.*cap*`
6. In file `utils/verify_dist_tarball.sh`:
	https://github.com/koalaman/shellcheck/wiki/SC2268
	`[ "x${TARBALL}" = x ]` → `[ -z "${TARBALL}" ]`
7. In file `utils/check_symbols.sh`:
	https://github.com/koalaman/shellcheck/wiki/SC2221
	`'[ndpi_utils.o]'|'[ndpi_memory.o]'|'[roaring.o]')` → `'[ndpi_utils.o]'|'[ndpi_memory.o]')`
8. In file `autogen.sh`:
	https://github.com/koalaman/shellcheck/wiki/SC2145
	`echo "./configure $@"` → `echo "./configure $*"`
	https://github.com/koalaman/shellcheck/wiki/SC2068
	`./configure $@` → `./configure "$@"`

III. `LIST6_MERGED` and `LIST_MERGED6`
	There were typos with this variables in files `utils/aws_ip_addresses_download.sh`, `utils/aws_ip_addresses_download.sh` and `utils/microsoft_ip_addresses_download.sh` where variable `LIST6_MERGED` was defined, but `LIST_MERGED6` was removed by `rm`.
	I changed all `LIST_MERGED6` to `LIST6_MERGED`.

Not all changes are absolutely necessary, but some may save you from future bugs.

autogen.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env sh
 
 rm -f configure config.h config.h.in
 
@@ -42,7 +42,6 @@ fi
 
 autoreconf -ivf
 
-echo "./configure $@"
+echo "./configure $*"
 chmod +x configure
-./configure $@
-
+./configure "$@"

packages/ubuntu/debian/postinst

@@ -1,4 +1,5 @@
-#!/bin/sh -e
+#!/usr/bin/env sh
+set -e
 
 case "$1" in
     configure)

packages/ubuntu/debian/postrm

@@ -1,5 +1,4 @@
-#!/bin/sh -e
-
+#!/usr/bin/env sh
 set -e
 
 /sbin/ldconfig

packages/ubuntu/debian/preinst

@@ -1,10 +1,9 @@
-#! /bin/sh
+#!/usr/bin/env sh
+set -e
 # preinst script
 #
 # see: dh_installdeb(1)
 
-set -e
-
 # summary of how this script can be called:
 #        * <new-preinst> `install'
 #        * <new-preinst> `install' <old-version>

packages/ubuntu/debian/prerm

@@ -1,4 +1,5 @@
-#!/bin/sh -e
+#!/usr/bin/env sh
+set -e
 
 case "$1" in
 upgrade)

packages/version.sh

@@ -1,9 +1,9 @@
-#!/bin/bash
+#!/usr/bin/env sh
 
-SCRIPTPATH="$(cd "$(dirname "$0")"; pwd -P)"
-RELEASE="$(cd ${SCRIPTPATH}; cat ../configure.ac|grep C_INIT|cut -c 20-|rev|cut -c 3-|rev)"
-MAJOR_RELEASE="$(cd ${SCRIPTPATH}; cat ../configure.ac|grep C_INIT|cut -c 20-|rev|cut -c 3-|rev|cut -d. -f1)"
-REVISION="$(cd ${SCRIPTPATH}; git rev-list --all |wc -l | tr -d '[[:space:]]')"
+SCRIPTPATH="$(cd "$(dirname "$0")" || exit 1; pwd -P)"
+RELEASE="$(cd "${SCRIPTPATH}" || exit 1; < ../configure.ac grep C_INIT | cut -c 20- | rev | cut -c 3- | rev)"
+MAJOR_RELEASE="$(cd "${SCRIPTPATH}" || exit 1; < ../configure.ac grep C_INIT | cut -c 20- | rev | cut -c 3- | rev | cut -d. -f1)"
+REVISION="$(cd "${SCRIPTPATH}" || exit 1; git rev-list --all | wc -l | tr -d '[[:space:]]')"
 
 get_release() {
 	echo "${RELEASE}"

tests/do-dga.sh

@@ -1,6 +1,6 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
-cd "$(dirname "${0}")"
+cd "$(dirname "${0}")" || exit 1
 
 # Baseline performances ------------------------------------------------------------------------------------------------
 # Important notes: BASE values must be integers examples and represents percentage (e.g. 79%, 98%).
@@ -18,9 +18,9 @@ DATA_SIZE=0
 RC=0
 
 get_evaluation_data_size() {
-  DGA_DATA_SIZE=`wc -l dga/test_dga.csv | awk '{split($0,a," "); print a[1]}'`
-  NON_DGA_DATA_SIZE=`wc -l dga/test_non_dga.csv | awk '{split($0,a," "); print a[1]}'`
-  DATA_SIZE=$(( $NON_DGA_DATA_SIZE + $DGA_DATA_SIZE ))
+  DGA_DATA_SIZE=$(wc -l dga/test_dga.csv | awk '{split($0,a," "); print a[1]}')
+  NON_DGA_DATA_SIZE=$(wc -l dga/test_non_dga.csv | awk '{split($0,a," "); print a[1]}')
+  DATA_SIZE=$(( NON_DGA_DATA_SIZE + DGA_DATA_SIZE ))
 }
 
 evaluate_ndpi_dga_detection() {
@@ -29,25 +29,25 @@ evaluate_ndpi_dga_detection() {
   # Precision: TP / (TP + FP)
   # Recall: TP / (TP + FN)
 
-  TP=`$DGA_EVALUATE dga/test_dga.csv`
-  FN=$(( $DGA_DATA_SIZE - $TP ))
-  FP=`$DGA_EVALUATE dga/test_non_dga.csv`
-  TN=$(( $NON_DGA_DATA_SIZE - $FP ))
+  TP=$($DGA_EVALUATE dga/test_dga.csv)
+  FN=$(( DGA_DATA_SIZE - TP ))
+  FP=$($DGA_EVALUATE dga/test_non_dga.csv)
+  TN=$(( NON_DGA_DATA_SIZE - FP ))
 
-  ACCURACY=`echo "print(int(((${TP} + ${TN})/(${TP} + ${TN} + ${FP} + ${FN}))*100))" | python3`
-  PRECISION=`echo "print(int(((${TP})/(${TP} + ${FP}))*100))" | python3`
-  RECALL=`echo "print(int(((${TP})/(${TP} + ${FN}))*100))" | python3`
+  ACCURACY=$(echo "print(int(((${TP} + ${TN})/(${TP} + ${TN} + ${FP} + ${FN}))*100))" | python3)
+  PRECISION=$(echo "print(int(((${TP})/(${TP} + ${FP}))*100))" | python3)
+  RECALL=$(echo "print(int(((${TP})/(${TP} + ${FN}))*100))" | python3)
 
   # In case modified version of classification algorithm decreases performances, test do not pass.
-  if [ $ACCURACY -lt $BASE_ACCURACY ]; then
+  if [ "$ACCURACY" -lt "$BASE_ACCURACY" ]; then
 		 printf "ERROR: Your modifications decreased DGA classifier accuracy: 0.${BASE_ACCURACY} decreased to 0.${ACCURACY}!\n"
 		 RC=1
   fi
-  if [ $PRECISION -lt $BASE_PRECISION ]; then
+  if [ "$PRECISION" -lt "$BASE_PRECISION" ]; then
 		 printf "ERROR: Your modifications decreased DGA classifier precision: 0.${BASE_PRECISION} decreased to 0.${PRECISION}!\n"
 		 RC=1
   fi
-  if [ $RECALL -lt $BASE_RECALL ]; then
+  if [ "$RECALL" -lt "$BASE_RECALL" ]; then
 		 printf "ERROR: Your modifications decreased DGA classifier recall: 0.${BASE_RECALL} decreased to 0.${RECALL}!\n"
 		 RC=1
   fi

tests/do-unit.sh

@@ -1,6 +1,6 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
-cd "$(dirname "${0}")"
+cd "$(dirname "${0}")" || exit 1
 
 UNIT="./unit/unit"
 

tests/do.sh.in

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 #
 # Use (to override results)
@@ -9,7 +9,7 @@
 #
 # NDPI_FORCE_PARALLEL_UTESTS=1 ./do.sh
 
-cd "$(dirname "${0}")"
+cd "$(dirname "${0}")" || exit 1
 
 FUZZY_TESTING_ENABLED=@BUILD_FUZZTARGETS@
 if [ "${NDPI_DISABLE_FUZZY}" = "1" ]; then
@@ -75,12 +75,12 @@ if [ ! -x "../example/ndpiReader${EXE_SUFFIX}" ]; then
   exit 1
 fi
 
-#For paralell tests you need `parallel` from GNU, not from `moreutils` package!
+#For parallel tests you need `parallel` from GNU, not from `moreutils` package!
 #On Ubuntu, for example, you might need to explicitly run `apt install parallel`
 if [ $FORCE_PARALLEL_UTESTS -eq 1 ]; then
   if ! parallel -V | grep -qoE 'GNU parallel'; then
-    echo "$0: To run the test in parallel mode you need **GNU** `parallel`"
-    echo "$0: Try something like `apt install parallel`"
+    echo "$0: To run the test in parallel mode you need **GNU** 'parallel'"
+    echo "$0: Try something like 'apt install parallel'"
     exit 1
   fi
 fi
@@ -114,22 +114,22 @@ run_single_pcap()
 {
     f=$1
 
-    if [ ! -f ./pcap/$f ]; then
+    if [ ! -f "./pcap/$f" ]; then
         return 0
     fi
 
     SKIP_PCAP=0;
     if [ $PCRE2_ENABLED -eq 0 ]; then
         for p in $PCRE_PCAPS; do
-            if [ $f = $p ]; then
+            if [ "$f" = "$p" ]; then
                 SKIP_PCAP=1
                 break
             fi
         done
     fi
     if [ $NBPF_ENABLED -eq 0 ]; then
         for p in $NBPF_PCAPS; do
-            if [ $f = $p ]; then
+            if [ "$f" = "$p" ]; then
                 SKIP_PCAP=1
                 break
             fi
@@ -149,10 +149,10 @@ run_single_pcap()
     CMD_RET=$?
     if [ $CMD_RET -eq 0 ] && [ -f /tmp/reader.$$.out ]; then
 	# create result files if not present
-        if [ ! -f result/$f.out ]; then
-            cp /tmp/reader.$$.out result/$f.out
+        if [ ! -f "result/$f.out" ]; then
+            cp /tmp/reader.$$.out "result/$f.out"
 	fi
-        NUM_DIFF=`${CMD_DIFF} result/$f.out /tmp/reader.$$.out | wc -l`
+        NUM_DIFF=$(${CMD_DIFF} "result/$f.out" /tmp/reader.$$.out | wc -l)
     else
         if [ $FORCE_PARALLEL_UTESTS -eq 1 ]; then
             printf "ERROR (ndpiReader${EXE_SUFFIX} exit code: ${CMD_RET})\n"
@@ -163,7 +163,7 @@ run_single_pcap()
 	return 1
     fi
 
-    if [ $NUM_DIFF -eq 0 ]; then
+    if [ "$NUM_DIFF" -eq 0 ]; then
         if [ $FORCE_PARALLEL_UTESTS -eq 0 ]; then
             printf "%-48s\tOK\n" "$f"
         fi
@@ -176,12 +176,12 @@ run_single_pcap()
 	    FAILURES+=("$f") #TODO: find a way to update this variable also in parallel mode
 	fi
         echo "$CMD [old vs new]"
-        ${CMD_DIFF} result/$f.out /tmp/reader.$$.out
-        if [ ! -z "${CMD_COLORDIFF}" -a ! -z "${CMD_WDIFF}" ]; then
-            ${CMD_WDIFF} -n -3 result/$f.out /tmp/reader.$$.out | sort | uniq | ${CMD_COLORDIFF}
+        ${CMD_DIFF} "result/$f.out" /tmp/reader.$$.out
+        if [ ! -z "${CMD_COLORDIFF}" ] && [ ! -z "${CMD_WDIFF}" ]; then
+            ${CMD_WDIFF} -n -3 "result/$f.out" /tmp/reader.$$.out | sort | uniq | ${CMD_COLORDIFF}
         fi
         if [ $FORCE_UPDATING_UTESTS_RESULTS -eq 1 ]; then
-            cp /tmp/reader.$$.out result/$f.out
+            cp /tmp/reader.$$.out "result/$f.out"
         fi
     fi
 
@@ -199,12 +199,12 @@ check_results() {
             parallel --bar --tag "run_single_pcap" ::: $PCAPS
         fi
 	RET=$? #Number of failed job up to 100
-	RC=$(( RC + $RET ))
+	RC=$(( RC + RET ))
     else
         for f in $PCAPS; do
-	    run_single_pcap $f
+	    run_single_pcap "$f"
 	    RET=$?
-	    RC=$(( RC + $RET ))
+	    RC=$(( RC + RET ))
         done
     fi
 
@@ -226,35 +226,35 @@ for d in $(find ./cfgs/* -type d -maxdepth 0 2>/dev/null) ; do
     SKIP_CFG=0
     if [ $GLOBAL_CONTEXT_ENABLED -eq 0 ]; then
         for c in $GLOBAL_CONTEXT_CFGS; do
-            if [ $c = $(basename $d) ]; then
+            if [ "$c" = "$(basename "$d")" ]; then
 	        SKIP_CFG=1
 	        break
 	    fi
         done
     fi
     if [ $SKIP_CFG -eq 1 ]; then
-        printf "Configuration \""$(basename $d)"\" %-18s\tSKIPPED\n"
+        printf "Configuration \"$(basename "$d")\" %-18s\tSKIPPED\n"
         continue
     fi
 
-    cd ./cfgs/"$(basename $d)"
+    cd ./cfgs/"$(basename "$d")" || exit 1
 
     if [ "$#" -ne 0 ]; then
         PCAPS=$*
     else
-        PCAPS=`cd pcap; /bin/ls *.*cap*`
+        PCAPS=$(cd pcap || exit 1; /bin/ls -- *.*cap*)
     fi
     FAILURES=()
     READER_EXTRA_PARAM=""
     [ -f config.txt ] && READER_EXTRA_PARAM=$(< config.txt)
     export READER_EXTRA_PARAM
 
-    echo "Run configuration \""$(basename $d)"\" [$READER_EXTRA_PARAM]"
+    echo "Run configuration \"$(basename "$d")\" [$READER_EXTRA_PARAM]"
 
     check_results
 
     test ${#FAILURES} -ne 0 && printf '%s: %s\n' "${0}" "${RC} pcap(s) failed"
-    test ${#FAILURES} -ne 0 && echo "Failed: ${FAILURES[@]}"
+    test ${#FAILURES} -ne 0 && echo "Failed: " "${FAILURES[@]}"
 
     cd ../../
 done

tests/ossfuzz.sh

@@ -1,4 +1,5 @@
-#!/bin/bash -eu
+#!/usr/bin/env bash
+set -eu
 # Copyright 2019 Google Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -46,25 +47,25 @@ cd ndpi
 RANLIB=llvm-ranlib LDFLAGS="-L/usr/local/lib -lpcap" ./autogen.sh --enable-fuzztargets --with-only-libndpi --enable-tls-sigs
 make -j$(nproc)
 # Copy fuzzers
-ls fuzz/fuzz* | grep -v "\." | while read i; do cp $i $OUT/; done
+ls fuzz/fuzz* | grep -v "\." | while read -r i; do cp "$i" "$OUT"/; done
 # Copy dictionaries
-cp fuzz/*.dict $OUT/
+cp fuzz/*.dict "$OUT"/
 # Copy seed corpus
-cp fuzz/*.zip $OUT/
+cp fuzz/*.zip "$OUT"/
 # Copy options
-cp fuzz/*.options $OUT/
+cp fuzz/*.options "$OUT"/
 # Copy configuration files
-cp example/protos.txt $OUT/
-cp example/categories.txt $OUT/
-cp example/risky_domains.txt $OUT/
-cp example/ja3_fingerprints.csv $OUT/
-cp example/sha1_fingerprints.csv $OUT/
-cp example/config.txt $OUT/
-cp lists/public_suffix_list.dat $OUT/
-cp fuzz/ipv*_addresses.txt $OUT/
-cp fuzz/bd_param.txt $OUT/
-cp fuzz/splt_param.txt $OUT/
-cp fuzz/random_list.list $OUT/
-mkdir -p $OUT/lists
+cp example/protos.txt "$OUT"/
+cp example/categories.txt "$OUT"/
+cp example/risky_domains.txt "$OUT"/
+cp example/ja3_fingerprints.csv "$OUT"/
+cp example/sha1_fingerprints.csv "$OUT"/
+cp example/config.txt "$OUT"/
+cp lists/public_suffix_list.dat "$OUT"/
+cp fuzz/ipv*_addresses.txt "$OUT"/
+cp fuzz/bd_param.txt "$OUT"/
+cp fuzz/splt_param.txt "$OUT"/
+cp fuzz/random_list.list "$OUT"/
+mkdir -p "$OUT"/lists
 # Ignore a huge list to speed up init time
-find lists/*.list ! -name 100_malware.list -exec cp -t $OUT/lists/ {} +
+find lists/*.list ! -name 100_malware.list -exec cp -t "$OUT"/lists/ {} +

utils/asn_update.sh

@@ -1,19 +1,18 @@
 #!/usr/bin/env bash
-
 set -e
 
 FAILED_ASN=0
 TOTAL_ASN=0
 
 function processing_list() {
 	local LIST_MERGED="/tmp/list_m"
-	local LIST_MERGED6="/tmp/list_m6"
+	local LIST6_MERGED="/tmp/list6_m"
 
 	echo "(2) Processing IP addresses..."
 	./mergeipaddrlist.py "$1" > $LIST_MERGED
-	./mergeipaddrlist.py "$2" > $LIST_MERGED6
-	./ipaddr2list.py "$LIST_MERGED" "$3" "$LIST_MERGED6" > "$4"
-	rm -f $LIST_MERGED
+	./mergeipaddrlist.py "$2" > $LIST6_MERGED
+	./ipaddr2list.py "$LIST_MERGED" "$3" "$LIST6_MERGED" > "$4"
+	rm -f $LIST_MERGED $LIST6_MERGED
 }
 
 function create_list() {
@@ -210,7 +209,7 @@ DEST=../src/lib/inc_generated/ndpi_asn_roblox.c.inc
 create_list NDPI_PROTOCOL_ROBLOX $DEST "" "AS22697"
 echo "(3) Roblox IPs are available in $DEST"
 
-if [ ${TOTAL_ASN} -eq 0 -o ${TOTAL_ASN} -eq ${FAILED_ASN} ]; then
+if [ ${TOTAL_ASN} -eq 0 ] || [ ${TOTAL_ASN} -eq ${FAILED_ASN} ]; then
 	printf '%s: %s\n' "${0}" "All download(s) failed, ./get_routes_by_asn.sh broken?"
 	exit 1
 else

utils/aws_ip_addresses_download.sh

@@ -1,5 +1,4 @@
 #!/usr/bin/env bash
-
 set -e
 
 cd "$(dirname "${0}")" || exit 1
@@ -31,7 +30,7 @@ is_file_empty "${LIST6_MERGED}"
 ./ipaddr2list.py $LIST_MERGED NDPI_PROTOCOL_AMAZON_AWS $LIST6_MERGED > $DEST
 is_file_empty "${DEST}"
 
-rm -f ${TMP} ${LIST} ${LIST6} ${LIST_MERGED} ${LIST_MERGED6}
+rm -f ${TMP} ${LIST} ${LIST6} ${LIST_MERGED} ${LIST6_MERGED}
 
 echo "(3) Amazon AWS IPs are available in $DEST"
 exit 0