Skip to content

Commit

Permalink
Keep track of file progress and host with separate variables
Browse files Browse the repository at this point in the history
Final commit with a host (!)
  • Loading branch information
alfonsomunozpomer committed Sep 21, 2023
1 parent f06c629 commit f9b9231
Showing 1 changed file with 12 additions and 7 deletions.
19 changes: 12 additions & 7 deletions bin/solr-jsonl-chunk-loader.sh
Original file line number Diff line number Diff line change
Expand Up @@ -66,23 +66,28 @@ cleanup() {


trap cleanup exit
I=O
# I is used to print the progress of a chunk file (e.g. 1/10)
# J is used to round-robin the hosts
I=0
J=0
for CHUNK_FILE in $CHUNK_FILES
do
# Round-robin the hosts
SOLR_HOST=${SOLR_HOSTS_ARR[(( ${I} % ${#SOLR_HOSTS_ARR[@]} ))]}
I=$(( $I + 1 ))

echo "$CHUNK_FILE ${I}/$(wc -w <<< $CHUNK_FILES)"

SOLR_HOST=${SOLR_HOSTS_ARR[(( ${J} % ${#SOLR_HOSTS_ARR[@]} ))]}
J=$(( $J + 1 ))
post_json ${SOLR_HOST} ${CHUNK_FILE}

if [[ $(( $I % ( $COMMIT_DOCS / $NUM_DOCS_PER_BATCH) )) == 0 ]]
then
# Make the commit in the next host
I=$(( $I + 1 ))
SOLR_HOST=${SOLR_HOSTS_ARR[(( ${I} % ${#SOLR_HOSTS_ARR[@]} ))]}
J=$(( $J + 1 ))
SOLR_HOST=${SOLR_HOSTS_ARR[(( ${J} % ${#SOLR_HOSTS_ARR[@]} ))]}
commit ${SOLR_HOST}
fi
done
commit

J=$(( $J + 1 ))
SOLR_HOST=${SOLR_HOSTS_ARR[(( ${J} % ${#SOLR_HOSTS_ARR[@]} ))]}
commit ${SOLR_HOST}

0 comments on commit f9b9231

Please sign in to comment.