From f9b9231001969536bd402026494c7e9d40a8d273 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Alfonso=20Mu=C3=B1oz-Pomer=20Fuentes?= <amunoz@ebi.ac.uk>
Date: Thu, 21 Sep 2023 16:55:39 +0100
Subject: [PATCH] Keep track of file progress and host with separate variables
 Final commit with a host (!)

---
 bin/solr-jsonl-chunk-loader.sh | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/bin/solr-jsonl-chunk-loader.sh b/bin/solr-jsonl-chunk-loader.sh
index 8ca407d..41fcd6d 100755
--- a/bin/solr-jsonl-chunk-loader.sh
+++ b/bin/solr-jsonl-chunk-loader.sh
@@ -66,23 +66,28 @@ cleanup() {
 
 
 trap cleanup exit
-I=O
+# I is used to print the progress of a chunk file (e.g. 1/10)
+# J is used to round-robin the hosts
+I=0
+J=0
 for CHUNK_FILE in $CHUNK_FILES
 do
-  # Round-robin the hosts
-  SOLR_HOST=${SOLR_HOSTS_ARR[(( ${I} % ${#SOLR_HOSTS_ARR[@]} ))]}
   I=$(( $I + 1 ))
-
   echo "$CHUNK_FILE ${I}/$(wc -w <<< $CHUNK_FILES)"
 
+  SOLR_HOST=${SOLR_HOSTS_ARR[(( ${J} % ${#SOLR_HOSTS_ARR[@]} ))]}
+  J=$(( $J + 1 ))
   post_json ${SOLR_HOST} ${CHUNK_FILE}
 
   if [[ $(( $I % ( $COMMIT_DOCS / $NUM_DOCS_PER_BATCH) )) == 0 ]]
   then
     # Make the commit in the next host
-    I=$(( $I + 1 ))
-    SOLR_HOST=${SOLR_HOSTS_ARR[(( ${I} % ${#SOLR_HOSTS_ARR[@]} ))]}
+    J=$(( $J + 1 ))
+    SOLR_HOST=${SOLR_HOSTS_ARR[(( ${J} % ${#SOLR_HOSTS_ARR[@]} ))]}
     commit ${SOLR_HOST}
   fi
 done
-commit
+
+J=$(( $J + 1 ))
+SOLR_HOST=${SOLR_HOSTS_ARR[(( ${J} % ${#SOLR_HOSTS_ARR[@]} ))]}
+commit ${SOLR_HOST}