diff --git a/run.sh b/run.sh index 5882125..6cc09f5 100755 --- a/run.sh +++ b/run.sh @@ -59,32 +59,32 @@ filter_list() { name="$2" shift 2 - mkdir -p "${name}.${network}.${CRAWL_DNS_DOMAIN}" - devp2p nodeset filter all.json -eth-network "$network" $@ > "${name}.${network}.${CRAWL_DNS_DOMAIN}/nodes.json" + mkdir -p "${name}.${network}.${CRAWL_DNS_DOMAIN}" || return 1 + devp2p nodeset filter all.json -eth-network "$network" $@ > "${name}.${network}.${CRAWL_DNS_DOMAIN}/nodes.json" || return 1 } generate_list() { - devp2p discv4 crawl -timeout "$CRAWL_TIMEOUT" all.json + devp2p discv4 crawl -timeout "$CRAWL_TIMEOUT" all.json || return 1 # Mainnet - filter_list mainnet all -limit 3000 - filter_list mainnet les -limit 200 -les-server - filter_list mainnet snap -limit 500 -snap + filter_list mainnet all -limit 3000 || return 1 + filter_list mainnet les -limit 200 -les-server || return 1 + filter_list mainnet snap -limit 500 -snap || return 1 # Sepolia - filter_list sepolia all -limit 250 - filter_list sepolia les -limit 25 -les-server - filter_list sepolia snap -limit 25 -snap + filter_list sepolia all -limit 250 || return 1 + filter_list sepolia les -limit 25 -les-server || return 1 + filter_list sepolia snap -limit 25 -snap || return 1 # Holesky - filter_list holesky all -limit 250 - filter_list holesky snap -limit 25 -snap + filter_list holesky all -limit 250 || return 1 + filter_list holesky snap -limit 25 -snap || return 1 } sign_lists() { for D in *."${CRAWL_DNS_DOMAIN}"; do if [ -d "${D}" ]; then - echo "" | devp2p dns sign "${D}" "$CRAWL_DNS_SIGNING_KEY" + echo "" | devp2p dns sign "${D}" "$CRAWL_DNS_SIGNING_KEY" || return 1 fi done } @@ -92,7 +92,7 @@ sign_lists() { publish_dns_cloudflare() { for D in *."${CRAWL_DNS_DOMAIN}"; do if [ -d "${D}" ]; then - devp2p dns to-cloudflare -zoneid "$CLOUDFLARE_ZONE_ID" "${D}" + devp2p dns to-cloudflare -zoneid "$CLOUDFLARE_ZONE_ID" "${D}" || return 1 fi done } @@ -100,21 +100,22 @@ publish_dns_cloudflare() { publish_dns_route53() { for D in *."${CRAWL_DNS_DOMAIN}"; do if [ -d "${D}" ]; then - devp2p dns to-route53 -zone-id "$ROUTE53_ZONE_ID" "${D}" + devp2p dns to-route53 -zone-id "$ROUTE53_ZONE_ID" "${D}" || return 1 fi done } git_push_crawler_output() { if [ -n "$(git status --porcelain)" ]; then - git add all.json ./*."${CRAWL_DNS_DOMAIN}"/*.json - git commit --message "automatic update: crawl time $CRAWL_TIMEOUT" - git push origin "$CRAWL_GIT_BRANCH" + git add all.json ./*."${CRAWL_DNS_DOMAIN}"/*.json || return 1 + git commit --message "automatic update: crawl time $CRAWL_TIMEOUT" || return 1 + git push origin "$CRAWL_GIT_BRANCH" || return 1 fi } publish_influx_metrics() { - echo -n "" > metrics.txt + local status=$1 + echo "devp2p_discv4.crawl_status value=${status}i" > metrics.txt for D in *."${CRAWL_DNS_DOMAIN}"; do if [ -d "${D}" ]; then LEN=$(jq length < "${D}/nodes.json") @@ -132,12 +133,13 @@ publish_influx_metrics() { init_prometheus_metrics() { go install -v github.com/projectdiscovery/simplehttpserver/cmd/simplehttpserver@v0.0.6 simplehttpserver -listen "${PROMETHEUS_METRICS_LISTEN}" -path "${prometheus_metrics_dir}" -silent & - publish_prometheus_metrics + publish_prometheus_metrics 0 # Init status with 0, which means it hasn't run yet } publish_prometheus_metrics() { + local status=$1 prometheus_metrics_file="${prometheus_metrics_dir}/metrics" - echo -n "" > "${prometheus_metrics_file}" + echo "devp2p_discv4_crawl_status ${status}" > "${prometheus_metrics_file}" for D in *."${CRAWL_DNS_DOMAIN}"; do if [ -d "${D}" ]; then LEN=$(jq length < "${D}/nodes.json") @@ -155,40 +157,45 @@ git_update_repo "$CRAWL_GIT_REPO" output "$CRAWL_GIT_BRANCH" PATH="$geth_src:$PATH" cd output -init_prometheus_metrics +if [ "$PROMETHEUS_METRICS_ENABLED" = true ] ; then + init_prometheus_metrics +fi while true do + # Initialize crawl status as success + crawl_status=1 + # Pull changes from go-ethereum. - update_devp2p_tool + update_devp2p_tool || crawl_status=0 # Generate node lists - generate_list + generate_list || crawl_status=0 # Sign lists if [ -f "$CRAWL_DNS_SIGNING_KEY" ]; then - sign_lists + sign_lists || crawl_status=0 fi # Push changes back to git repo if [ "$CRAWL_GIT_PUSH" = true ] ; then - git_push_crawler_output + git_push_crawler_output || crawl_status=0 fi # Publish DNS records if [ "$CRAWL_DNS_PUBLISH_CLOUDFLARE" = true ] ; then - publish_dns_cloudflare + publish_dns_cloudflare || crawl_status=0 fi if [ "$CRAWL_DNS_PUBLISH_ROUTE53" = true ] ; then - publish_dns_route53 + publish_dns_route53 || crawl_status=0 fi # Publish metrics if [ "$INFLUXDB_METRICS_ENABLED" = true ] ; then - publish_influx_metrics + publish_influx_metrics $crawl_status fi if [ "$PROMETHEUS_METRICS_ENABLED" = true ] ; then - publish_prometheus_metrics + publish_prometheus_metrics $crawl_status fi if [ "$CRAWL_RUN_ONCE" = true ] ; then