Skip to content

Commit

Permalink
feat: add Geth restore pre-task
Browse files Browse the repository at this point in the history
  • Loading branch information
mrekucci committed Aug 27, 2024
1 parent 6441dae commit fd5afa5
Showing 1 changed file with 116 additions and 9 deletions.
125 changes: 116 additions & 9 deletions infrastructure/nomad/playbooks/templates/jobs/mev-commit-geth.nomad.j2
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,110 @@ job "{{ job.name }}" {
read_only = false
}

task "restore" {
driver = "exec"

lifecycle {
hook = "prestart"
}

{% if env == 'testenv' %}
resources {
cores = 6
memory = 32768
}
{% endif %}

volume_mount {
volume = "tmp-volume"
destination = "/local/backups"
read_only = false
}

{% if env != 'devenv' %}
artifact {
source = "https://primev-infrastructure-artifacts.s3.us-west-2.amazonaws.com/genesis_{{ version }}.json"
}
artifact {
source = "https://primev-infrastructure-artifacts.s3.us-west-2.amazonaws.com/mev-commit-geth_{{ version }}_Linux_{{ target_system_architecture }}.tar.gz"
}
{% else %}
artifact {
source = "http://{{ ansible_facts['default_ipv4']['address'] }}:1111/genesis_{{ version }}.json"
}
artifact {
source = "http://{{ ansible_facts['default_ipv4']['address'] }}:1111/mev-commit-geth_{{ version }}_Linux_{{ target_system_architecture }}.tar.gz"
}
{% endif %}

template {
data = <<-EOH
{%- raw %}
GETH_DATA_DIR="/alloc/data/node-{{ env "NOMAD_ALLOC_INDEX" }}"
{% endraw %}
GENESIS_L1_PATH="local/genesis_{{ version }}.json"
EOH
destination = "secrets/.env"
env = true
}

template {
data = <<-EOH
#!/usr/bin/env bash

BACKUP_FILE="local/backups/$(
ls -1 local/backups/ |
sed -r 's/(.*)_dirty/\1/' |
awk -v jobname="{{ job.name }}-{% raw %}{{ env "NOMAD_ALLOC_INDEX" }}{% endraw %}" -F'_' '$2 == jobname {print $3, $0}' |
sort -k1,1r |
head -n 1 |
cut -d' ' -f2
)"
if [[ ! -f "${BACKUP_FILE}" ]]; then
echo "No backup found, skipping restore"
exit 0
fi

chmod +x local/mev-commit-geth
CHAIN_ID=$(cat "${GENESIS_L1_PATH}" | jq -r .config.chainId)

if [[ ! -d "${GETH_DATA_DIR}/geth/chaindata" ]]; then
echo "Initializing chain data directory"
local/mev-commit-geth \
--db.engine=pebble \
--state.scheme=path \
--datadir="${GETH_DATA_DIR}" \
init "${GENESIS_L1_PATH}"
else
echo "Chain data directory already exists, skipping restore"
exit 0
fi

echo "Restoring from backup file ${BACKUP_FILE} to ${GETH_DATA_DIR}"
echo "Backup file size: $(du -h ${BACKUP_FILE} | cut -f1)"
local/mev-commit-geth \
--cache=4096 \
--verbosity=5 \
--datadir="${GETH_DATA_DIR}" \
import ${BACKUP_FILE}

if [[ $? -eq 0 ]]; then
echo "Restore successful"
else
echo "Restore failed"
exit 1
fi
EOH
destination = "local/run.sh"
perms = "0755"
}

config {
command = "bash"
args = ["-c", "local/run.sh"]
}
}

task "node" {
driver = "exec"
kill_timeout = "25s"
Expand Down Expand Up @@ -228,22 +332,25 @@ job "{{ job.name }}" {
STATUS=$(nomad alloc status -address="http://{{ ansible_facts['default_ipv4']['address'] }}:4646" -json "${NOMAD_ALLOC_ID}")
NON_ZERO_EXIT_EVENTS=$(echo "$STATUS" | jq -r '.TaskStates.node.Events[] | select(.ExitCode != 0)')
if [[ -n "${NON_ZERO_EXIT_EVENTS}" ]]; then
echo "The main task did not start or finish gracefully."
echo "The main task did not start or finish gracefully"
BACKUP_FILE+="-dirty"
fi
BACKUP_FILE+=".tar.gz"
BACKUP_FILE+=".rlp"

echo "Exporting chain data from ${GETH_DATA_DIR} to backup file: ${BACKUP_FILE}"
chmod +x local/mev-commit-geth
local/mev-commit-geth export \
local/mev-commit-geth \
--cache 4096 \
--syncmode "full" \
--datadir ${GETH_DATA_DIR} \
${BACKUP_FILE}
--verbosity=5 \
--datadir="${GETH_DATA_DIR}" \
export ${BACKUP_FILE}

if [[ -f "${BACKUP_FILE}" ]]; then
echo "Backup successful."
if [[ "$?" -eq 0 ]] && [[ -f "${BACKUP_FILE}" ]]; then
echo "Backup successful"
echo "Backup file size: $(du -h ${BACKUP_FILE} | cut -f1)"
else
echo "Backup failed."
echo "Backup failed"
exit 1
fi
EOH
destination = "local/run.sh"
Expand Down

0 comments on commit fd5afa5

Please sign in to comment.