feat: add status metric to detect failed runs (#10)

Previously we couldn't identify if a run failed just based on the
reported nodes. Because these were based on the contents of the files.
If a run would fail, the file could still exist and it would report a
valid ENR count.

This additional metric should make it easy to spot if a run failed.

run.sh

@@ -59,62 +59,63 @@ filter_list() {
   name="$2"
   shift 2
 
-  mkdir -p "${name}.${network}.${CRAWL_DNS_DOMAIN}"
-  devp2p nodeset filter all.json -eth-network "$network" $@ > "${name}.${network}.${CRAWL_DNS_DOMAIN}/nodes.json"
+  mkdir -p "${name}.${network}.${CRAWL_DNS_DOMAIN}" || return 1
+  devp2p nodeset filter all.json -eth-network "$network" $@ > "${name}.${network}.${CRAWL_DNS_DOMAIN}/nodes.json" || return 1
 }
 
 generate_list() {
-  devp2p discv4 crawl -timeout "$CRAWL_TIMEOUT" all.json
+  devp2p discv4 crawl -timeout "$CRAWL_TIMEOUT" all.json || return 1
 
   # Mainnet
-  filter_list mainnet all   -limit 3000
-  filter_list mainnet les   -limit 200  -les-server
-  filter_list mainnet snap  -limit 500  -snap
+  filter_list mainnet all   -limit 3000 || return 1
+  filter_list mainnet les   -limit 200  -les-server || return 1
+  filter_list mainnet snap  -limit 500  -snap || return 1
 
   # Sepolia
-  filter_list sepolia all   -limit 250
-  filter_list sepolia les   -limit 25   -les-server
-  filter_list sepolia snap  -limit 25   -snap
+  filter_list sepolia all   -limit 250 || return 1
+  filter_list sepolia les   -limit 25   -les-server || return 1
+  filter_list sepolia snap  -limit 25   -snap || return 1
 
   # Holesky
-  filter_list holesky all   -limit 250
-  filter_list holesky snap  -limit 25   -snap
+  filter_list holesky all   -limit 250 || return 1
+  filter_list holesky snap  -limit 25   -snap || return 1
 }
 
 sign_lists() {
   for D in *."${CRAWL_DNS_DOMAIN}"; do
     if [ -d "${D}" ]; then
-      echo "" | devp2p dns sign "${D}" "$CRAWL_DNS_SIGNING_KEY"
+      echo "" | devp2p dns sign "${D}" "$CRAWL_DNS_SIGNING_KEY" || return 1
     fi
   done
 }
 
 publish_dns_cloudflare() {
   for D in *."${CRAWL_DNS_DOMAIN}"; do
     if [ -d "${D}" ]; then
-      devp2p dns to-cloudflare -zoneid "$CLOUDFLARE_ZONE_ID" "${D}"
+      devp2p dns to-cloudflare -zoneid "$CLOUDFLARE_ZONE_ID" "${D}" || return 1
     fi
   done
 }
 
 publish_dns_route53() {
   for D in *."${CRAWL_DNS_DOMAIN}"; do
     if [ -d "${D}" ]; then
-      devp2p dns to-route53 -zone-id "$ROUTE53_ZONE_ID" "${D}"
+      devp2p dns to-route53 -zone-id "$ROUTE53_ZONE_ID" "${D}" || return 1
     fi
   done
 }
 
 git_push_crawler_output() {
   if [ -n "$(git status --porcelain)" ]; then
-    git add all.json ./*."${CRAWL_DNS_DOMAIN}"/*.json
-    git commit --message "automatic update: crawl time $CRAWL_TIMEOUT"
-    git push origin "$CRAWL_GIT_BRANCH"
+    git add all.json ./*."${CRAWL_DNS_DOMAIN}"/*.json || return 1
+    git commit --message "automatic update: crawl time $CRAWL_TIMEOUT" || return 1
+    git push origin "$CRAWL_GIT_BRANCH" || return 1
   fi
 }
 
 publish_influx_metrics() {
-  echo -n "" > metrics.txt
+  local status=$1
+  echo "devp2p_discv4.crawl_status value=${status}i" > metrics.txt
   for D in *."${CRAWL_DNS_DOMAIN}"; do
     if [ -d "${D}" ]; then
       LEN=$(jq length < "${D}/nodes.json")
@@ -132,12 +133,13 @@ publish_influx_metrics() {
 init_prometheus_metrics() {
   go install -v github.com/projectdiscovery/simplehttpserver/cmd/[email protected]
   simplehttpserver -listen "${PROMETHEUS_METRICS_LISTEN}" -path "${prometheus_metrics_dir}" -silent &
-  publish_prometheus_metrics
+  publish_prometheus_metrics 0 # Init status with 0, which means it hasn't run yet
 }
 
 publish_prometheus_metrics() {
+  local status=$1
   prometheus_metrics_file="${prometheus_metrics_dir}/metrics"
-  echo -n "" > "${prometheus_metrics_file}"
+  echo "devp2p_discv4_crawl_status ${status}" > "${prometheus_metrics_file}"
   for D in *."${CRAWL_DNS_DOMAIN}"; do
     if [ -d "${D}" ]; then
       LEN=$(jq length < "${D}/nodes.json")
@@ -155,40 +157,45 @@ git_update_repo "$CRAWL_GIT_REPO" output "$CRAWL_GIT_BRANCH"
 PATH="$geth_src:$PATH"
 cd output
 
-init_prometheus_metrics
+if [ "$PROMETHEUS_METRICS_ENABLED" = true ] ; then
+  init_prometheus_metrics
+fi
 
 while true
 do
+  # Initialize crawl status as success
+  crawl_status=1
+
   # Pull changes from go-ethereum.
-  update_devp2p_tool
+  update_devp2p_tool || crawl_status=0
 
   # Generate node lists
-  generate_list
+  generate_list || crawl_status=0
 
   # Sign lists
   if [ -f "$CRAWL_DNS_SIGNING_KEY" ]; then
-    sign_lists
+    sign_lists || crawl_status=0
   fi
 
   # Push changes back to git repo
   if [ "$CRAWL_GIT_PUSH" = true ] ; then
-    git_push_crawler_output
+    git_push_crawler_output || crawl_status=0
   fi
 
   # Publish DNS records
   if [ "$CRAWL_DNS_PUBLISH_CLOUDFLARE" = true ] ; then
-    publish_dns_cloudflare
+    publish_dns_cloudflare || crawl_status=0
   fi
   if [ "$CRAWL_DNS_PUBLISH_ROUTE53" = true ] ; then
-    publish_dns_route53
+    publish_dns_route53 || crawl_status=0
   fi
 
   # Publish metrics
   if [ "$INFLUXDB_METRICS_ENABLED" = true ] ; then
-    publish_influx_metrics
+    publish_influx_metrics $crawl_status
   fi
   if [ "$PROMETHEUS_METRICS_ENABLED" = true ] ; then
-    publish_prometheus_metrics
+    publish_prometheus_metrics $crawl_status
   fi
 
   if [ "$CRAWL_RUN_ONCE" = true ] ; then