Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add exponential backoff retry logic on initial connection #581

Closed
wants to merge 3 commits into from
+108 −93
Closed

Add exponential backoff retry logic on initial connection #581

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d687673
adding exponential backoff function + formatting helper_functions.sh
aarro Jul 13, 2024
7b475e6
Merge branch 'add-exponential-backoff'
aarro Jul 13, 2024
a8014ef
Merge branch 'main' into main
thijsvanloef Aug 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
144 changes: 80 additions & 64 deletions scripts/helper_functions.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,11 +79,11 @@ isExecutable() {
# Convert player list from JSON format
convert_JSON_to_CSV_players() {
echo 'name,playeruid,steamid'
echo -n "${1}" | \
jq -r '.players[] | [ .name, .playerId, .userId ] | @csv' | \
echo -n "${1}" |
jq -r '.players[] | [ .name, .playerId, .userId ] | @csv' |
sed -re 's/"None"/"00000000000000000000000000000000"/' \
-re 's/"steam_/"/' \
-re 's/"//g'
-re 's/"steam_/"/' \
-re 's/"//g'
}

# Lists players
Expand Down Expand Up @@ -120,56 +120,56 @@ get_player_count() {
# Log Definitions
#
export LINE='\n'
export RESET='\033[0m' # Text Reset
export WhiteText='\033[0;37m' # White
export RESET='\033[0m' # Text Reset
export WhiteText='\033[0;37m' # White

# Bold
export RedBoldText='\033[1;31m' # Red
export GreenBoldText='\033[1;32m' # Green
export YellowBoldText='\033[1;33m' # Yellow
export CyanBoldText='\033[1;36m' # Cyan
export RedBoldText='\033[1;31m' # Red
export GreenBoldText='\033[1;32m' # Green
export YellowBoldText='\033[1;33m' # Yellow
export CyanBoldText='\033[1;36m' # Cyan

LogInfo() {
Log "$1" "$WhiteText"
Log "$1" "$WhiteText"
}
LogWarn() {
Log "$1" "$YellowBoldText"
Log "$1" "$YellowBoldText"
}
LogError() {
Log "$1" "$RedBoldText"
Log "$1" "$RedBoldText"
}
LogSuccess() {
Log "$1" "$GreenBoldText"
Log "$1" "$GreenBoldText"
}
LogAction() {
Log "$1" "$CyanBoldText" "****" "****"
Log "$1" "$CyanBoldText" "****" "****"
}
Log() {
local message="$1"
local color="$2"
local prefix="$3"
local suffix="$4"
printf "$color%s$RESET$LINE" "$prefix$message$suffix"
local message="$1"
local color="$2"
local prefix="$3"
local suffix="$4"
printf "$color%s$RESET$LINE" "$prefix$message$suffix"
}

# Send Discord Message
# Level is optional variable defaulting to info
DiscordMessage() {
local title="$1"
local message="$2"
local level="$3"
local enabled="$4"
local webhook_url="$5"
if [ -z "$level" ]; then
level="info"
fi
if [ -n "${DISCORD_WEBHOOK_URL}" ]; then
/home/steam/server/discord.sh "$title" "$message" "$level" "$enabled" "$webhook_url" &
fi
local title="$1"
local message="$2"
local level="$3"
local enabled="$4"
local webhook_url="$5"
if [ -z "$level" ]; then
level="info"
fi
if [ -n "${DISCORD_WEBHOOK_URL}" ]; then
/home/steam/server/discord.sh "$title" "$message" "$level" "$enabled" "$webhook_url" &
fi
}

# REST API Call
REST_API(){
REST_API() {
local -r api="${1}"
local -r data="${2}"
local -r url="http://localhost:${REST_API_PORT}/v1/api/${api}"
Expand All @@ -178,7 +178,7 @@ REST_API(){
local -r post_api="save|stop"
local -i result=0
if [ "${data}" = "" ] && [[ ! ${api} =~ ${post_api} ]]; then
curl -s -L -X GET "${url}" -H "${accept}" -u "${userpass}"
curl -s -L -X GET "${url}" -H "${accept}" -u "${userpass}"
result=$?
else
curl -s -L -X POST "${url}" -H "${accept}" -u "${userpass}" --json "${data}"
Expand All @@ -189,8 +189,8 @@ REST_API(){

# RCON Call
RCON() {
local args="$1"
rcon-cli -c /home/steam/server/rcon.yaml "$args"
local args="$1"
rcon-cli -c /home/steam/server/rcon.yaml "$args"
}

# Given a message this will broadcast in game
Expand All @@ -212,7 +212,7 @@ broadcast_command() {
if [[ $TEXT = *[![:ascii:]]* ]]; then
LogWarn "Unable to broadcast since the message contains non-ascii characters: \"${message}\""
return_val=1
elif ! RCON "broadcast ${message}" > /dev/null; then
elif ! RCON "broadcast ${message}" >/dev/null; then
return_val=1
fi
return "$return_val"
Expand Down Expand Up @@ -258,35 +258,31 @@ countdown_message() {
# Only do countdown if there are players
if [ "$(get_player_count)" -gt 0 ]; then
if [[ "${mtime}" =~ ^[0-9]+$ ]]; then
for ((i = "${mtime}" ; i > 0 ; i--)); do
for ((i = "${mtime}"; i > 0; i--)); do
case "$i" in
1 )
broadcast_command "${message_prefix} in ${i} minute"
sleep 30s
broadcast_command "${message_prefix} in 30 seconds"
sleep 20s
broadcast_command "${message_prefix} in 10 seconds"
sleep 10s
;;
2 )
;&
3 )
;&
10 )
;&
15 )
;&
"$mtime" )
broadcast_command "${message_prefix} in ${i} minutes"
;&
* )
sleep 1m
# Checking for players every minute
# Checking after sleep since it is ran in the beginning of the function
if [ "$(get_player_count)" -eq 0 ]; then
break
fi
;;
1)
broadcast_command "${message_prefix} in ${i} minute"
sleep 30s
broadcast_command "${message_prefix} in 30 seconds"
sleep 20s
broadcast_command "${message_prefix} in 10 seconds"
sleep 10s
;;
2) ;&
3) ;&
10) ;&
15) ;&
"$mtime")
broadcast_command "${message_prefix} in ${i} minutes"
;&
*)
sleep 1m
# Checking for players every minute
# Checking after sleep since it is ran in the beginning of the function
if [ "$(get_player_count)" -eq 0 ]; then
break
fi
;;
esac
done
# If there are players but mtime is empty
Expand Down Expand Up @@ -325,3 +321,23 @@ get_latest_version() {
echo "$latest_version"
}

MAX_RETRIES=5
START_DELAY=2
MAX_DELAY=32

# https://www.deploymastery.com/2023/05/24/how-to-implement-exponential-backoff-in-bash/
function exponential_backoff {
local delay=$START_DELAY
for i in $(seq 1 "$MAX_RETRIES"); do
# Try the operation
if "$@"; then
return 0
fi

if [[ $i -lt "$MAX_RETRIES" ]]; then
sleep "$delay"
delay=$((delay * 2))
fi
done
return 1
}
57 changes: 28 additions & 29 deletions scripts/start.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ fi

# Update Only If Already Installed
if [ "$ServerInstalled" == 0 ] && [ "${UPDATE_ON_BOOT,,}" == true ]; then
UpdateRequired
exponential_backoff UpdateRequired
IsUpdateRequired=$?
if [ "$IsUpdateRequired" == 0 ]; then
LogAction "Starting Update"
Expand All @@ -37,15 +37,14 @@ fi
if [ "$architecture" == "arm64" ]; then
# create an arm64 version of ./PalServer.sh
cp ./PalServer.sh ./PalServer-arm64.sh

sed -i "s|\(\"\$UE_PROJECT_ROOT\/Pal\/Binaries\/Linux\/PalServer-Linux-Shipping\" Pal \"\$@\"\)|LD_LIBRARY_PATH=/home/steam/steamcmd/linux64:\$LD_LIBRARY_PATH /usr/local/bin/box64 \1|" ./PalServer-arm64.sh
chmod +x ./PalServer-arm64.sh
STARTCOMMAND=("./PalServer-arm64.sh")
else
STARTCOMMAND=("./PalServer.sh")
fi


#Validate Installation
if ! fileExists "${STARTCOMMAND[0]}"; then
LogError "Server Not Installed Properly"
Expand Down Expand Up @@ -76,54 +75,54 @@ LogAction "Checking for available container updates"
container_version_check

if [ "${DISABLE_GENERATE_SETTINGS,,}" = true ]; then
LogAction "GENERATING CONFIG"
LogWarn "Env vars will not be applied due to DISABLE_GENERATE_SETTINGS being set to TRUE!"

# shellcheck disable=SC2143
if [ ! "$(grep -s '[^[:space:]]' /palworld/Pal/Saved/Config/LinuxServer/PalWorldSettings.ini)" ]; then
LogAction "GENERATING CONFIG"
# Server will generate all ini files after first run.
if [ "$architecture" == "arm64" ]; then
timeout --preserve-status 15s ./PalServer-arm64.sh 1> /dev/null
else
timeout --preserve-status 15s ./PalServer.sh 1> /dev/null
fi

# Wait for shutdown
sleep 5
cp /palworld/DefaultPalWorldSettings.ini /palworld/Pal/Saved/Config/LinuxServer/PalWorldSettings.ini
fi
LogAction "GENERATING CONFIG"
LogWarn "Env vars will not be applied due to DISABLE_GENERATE_SETTINGS being set to TRUE!"

# shellcheck disable=SC2143
if [ ! "$(grep -s '[^[:space:]]' /palworld/Pal/Saved/Config/LinuxServer/PalWorldSettings.ini)" ]; then
LogAction "GENERATING CONFIG"
# Server will generate all ini files after first run.
if [ "$architecture" == "arm64" ]; then
timeout --preserve-status 15s ./PalServer-arm64.sh 1>/dev/null
else
timeout --preserve-status 15s ./PalServer.sh 1>/dev/null
fi

# Wait for shutdown
sleep 5
cp /palworld/DefaultPalWorldSettings.ini /palworld/Pal/Saved/Config/LinuxServer/PalWorldSettings.ini
fi
else
LogAction "GENERATING CONFIG"
LogInfo "Using Env vars to create PalWorldSettings.ini"
/home/steam/server/compile-settings.sh || exit
LogAction "GENERATING CONFIG"
LogInfo "Using Env vars to create PalWorldSettings.ini"
/home/steam/server/compile-settings.sh || exit
fi

if [ "${DISABLE_GENERATE_ENGINE,,}" = false ]; then
/home/steam/server/compile-engine.sh || exit
fi

LogAction "GENERATING CRONTAB"
truncate -s 0 "/home/steam/server/crontab"
truncate -s 0 "/home/steam/server/crontab"

if [ "${BACKUP_ENABLED,,}" = true ]; then
LogInfo "BACKUP_ENABLED=${BACKUP_ENABLED,,}"
LogInfo "Adding cronjob for auto backups"
echo "$BACKUP_CRON_EXPRESSION bash /usr/local/bin/backup" >> "/home/steam/server/crontab"
echo "$BACKUP_CRON_EXPRESSION bash /usr/local/bin/backup" >>"/home/steam/server/crontab"
supercronic -quiet -test "/home/steam/server/crontab" || exit
fi

if [ "${AUTO_UPDATE_ENABLED,,}" = true ] && [ "${UPDATE_ON_BOOT}" = true ]; then
LogInfo "AUTO_UPDATE_ENABLED=${AUTO_UPDATE_ENABLED,,}"
LogInfo "Adding cronjob for auto updating"
echo "$AUTO_UPDATE_CRON_EXPRESSION bash /usr/local/bin/update" >> "/home/steam/server/crontab"
echo "$AUTO_UPDATE_CRON_EXPRESSION bash /usr/local/bin/update" >>"/home/steam/server/crontab"
supercronic -quiet -test "/home/steam/server/crontab" || exit
fi

if [ "${AUTO_REBOOT_ENABLED,,}" = true ] && [ "${RCON_ENABLED,,}" = true ]; then
LogInfo "AUTO_REBOOT_ENABLED=${AUTO_REBOOT_ENABLED,,}"
LogInfo "Adding cronjob for auto rebooting"
echo "$AUTO_REBOOT_CRON_EXPRESSION bash /home/steam/server/auto_reboot.sh" >> "/home/steam/server/crontab"
echo "$AUTO_REBOOT_CRON_EXPRESSION bash /home/steam/server/auto_reboot.sh" >>"/home/steam/server/crontab"
supercronic -quiet -test "/home/steam/server/crontab" || exit
fi

Expand All @@ -135,13 +134,13 @@ else
fi

# Configure RCON settings
cat >/home/steam/server/rcon.yaml <<EOL
cat >/home/steam/server/rcon.yaml <<EOL
default:
address: "127.0.0.1:${RCON_PORT}"
password: "${ADMIN_PASSWORD}"
EOL

if [ "${ENABLE_PLAYER_LOGGING,,}" = true ] && [[ "${PLAYER_LOGGING_POLL_PERIOD}" =~ ^[0-9]+$ ]] && { [ "${REST_API_ENABLED,,}" = true ] || [ "${RCON_ENABLED,,}" = true ] ;} then
if [ "${ENABLE_PLAYER_LOGGING,,}" = true ] && [[ "${PLAYER_LOGGING_POLL_PERIOD}" =~ ^[0-9]+$ ]] && { [ "${REST_API_ENABLED,,}" = true ] || [ "${RCON_ENABLED,,}" = true ]; }; then
if [[ "$(id -u)" -eq 0 ]]; then
su steam -c /home/steam/server/player_logging.sh &
else
Expand Down
Loading