From fdff6748aaa91a0ea5bde803f0e773ada95ca025 Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Tue, 3 Dec 2024 09:50:40 +0100 Subject: [PATCH 1/3] fix: add status metric to detect failed runs --- run.sh | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/run.sh b/run.sh index 991b39b..51b7beb 100755 --- a/run.sh +++ b/run.sh @@ -71,11 +71,6 @@ generate_list() { filter_list mainnet les -limit 200 -les-server filter_list mainnet snap -limit 500 -snap - # Goerli - filter_list goerli all -limit 500 - filter_list goerli les -limit 50 -les-server - filter_list goerli snap -limit 50 -snap - # Sepolia filter_list sepolia all -limit 250 filter_list sepolia les -limit 25 -les-server @@ -119,7 +114,8 @@ git_push_crawler_output() { } publish_influx_metrics() { - echo -n "" > metrics.txt + local status=$1 + echo "devp2p_discv4.dns_node_crawl_status value=${status}i" > metrics.txt for D in *."${CRAWL_DNS_DOMAIN}"; do if [ -d "${D}" ]; then LEN=$(jq length < "${D}/nodes.json") @@ -141,8 +137,9 @@ init_prometheus_metrics() { } publish_prometheus_metrics() { + local status=$1 prometheus_metrics_file="${prometheus_metrics_dir}/metrics" - echo -n "" > "${prometheus_metrics_file}" + echo "devp2p_discv4_dns_nodes_crawl_status ${status}" > "${prometheus_metrics_file}" for D in *."${CRAWL_DNS_DOMAIN}"; do if [ -d "${D}" ]; then LEN=$(jq length < "${D}/nodes.json") @@ -160,40 +157,45 @@ git_update_repo "$CRAWL_GIT_REPO" output "$CRAWL_GIT_BRANCH" PATH="$geth_src:$PATH" cd output -init_prometheus_metrics +if [ "$PROMETHEUS_METRICS_ENABLED" = true ] ; then + init_prometheus_metrics +fi while true do + # Initialize crawl status as success + crawl_status=1 + # Pull changes from go-ethereum. - update_devp2p_tool + update_devp2p_tool || crawl_status=0 # Generate node lists - generate_list + generate_list || crawl_status=0 # Sign lists if [ -f "$CRAWL_DNS_SIGNING_KEY" ]; then - sign_lists + sign_lists || crawl_status=0 fi # Push changes back to git repo if [ "$CRAWL_GIT_PUSH" = true ] ; then - git_push_crawler_output + git_push_crawler_output || crawl_status=0 fi # Publish DNS records if [ "$CRAWL_DNS_PUBLISH_CLOUDFLARE" = true ] ; then - publish_dns_cloudflare + publish_dns_cloudflare || crawl_status=0 fi if [ "$CRAWL_DNS_PUBLISH_ROUTE53" = true ] ; then - publish_dns_route53 + publish_dns_route53 || crawl_status=0 fi # Publish metrics if [ "$INFLUXDB_METRICS_ENABLED" = true ] ; then - publish_influx_metrics + publish_influx_metrics $crawl_status fi if [ "$PROMETHEUS_METRICS_ENABLED" = true ] ; then - publish_prometheus_metrics + publish_prometheus_metrics $crawl_status fi if [ "$CRAWL_RUN_ONCE" = true ] ; then From 23eda3f4bde079b06bfad68a51675d819a7f103e Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Tue, 3 Dec 2024 11:15:02 +0100 Subject: [PATCH 2/3] chore: rename metric --- run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run.sh b/run.sh index 51b7beb..8fd0d86 100755 --- a/run.sh +++ b/run.sh @@ -115,7 +115,7 @@ git_push_crawler_output() { publish_influx_metrics() { local status=$1 - echo "devp2p_discv4.dns_node_crawl_status value=${status}i" > metrics.txt + echo "devp2p_discv4.crawl_status value=${status}i" > metrics.txt for D in *."${CRAWL_DNS_DOMAIN}"; do if [ -d "${D}" ]; then LEN=$(jq length < "${D}/nodes.json") @@ -139,7 +139,7 @@ init_prometheus_metrics() { publish_prometheus_metrics() { local status=$1 prometheus_metrics_file="${prometheus_metrics_dir}/metrics" - echo "devp2p_discv4_dns_nodes_crawl_status ${status}" > "${prometheus_metrics_file}" + echo "devp2p_discv4_crawl_status ${status}" > "${prometheus_metrics_file}" for D in *."${CRAWL_DNS_DOMAIN}"; do if [ -d "${D}" ]; then LEN=$(jq length < "${D}/nodes.json") From d271d9ad489cc4f191c47038ac8f07751eae6857 Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Tue, 3 Dec 2024 12:55:40 +0100 Subject: [PATCH 3/3] fix: return errors --- run.sh | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/run.sh b/run.sh index 8fd0d86..6cc09f5 100755 --- a/run.sh +++ b/run.sh @@ -59,32 +59,32 @@ filter_list() { name="$2" shift 2 - mkdir -p "${name}.${network}.${CRAWL_DNS_DOMAIN}" - devp2p nodeset filter all.json -eth-network "$network" $@ > "${name}.${network}.${CRAWL_DNS_DOMAIN}/nodes.json" + mkdir -p "${name}.${network}.${CRAWL_DNS_DOMAIN}" || return 1 + devp2p nodeset filter all.json -eth-network "$network" $@ > "${name}.${network}.${CRAWL_DNS_DOMAIN}/nodes.json" || return 1 } generate_list() { - devp2p discv4 crawl -timeout "$CRAWL_TIMEOUT" all.json + devp2p discv4 crawl -timeout "$CRAWL_TIMEOUT" all.json || return 1 # Mainnet - filter_list mainnet all -limit 3000 - filter_list mainnet les -limit 200 -les-server - filter_list mainnet snap -limit 500 -snap + filter_list mainnet all -limit 3000 || return 1 + filter_list mainnet les -limit 200 -les-server || return 1 + filter_list mainnet snap -limit 500 -snap || return 1 # Sepolia - filter_list sepolia all -limit 250 - filter_list sepolia les -limit 25 -les-server - filter_list sepolia snap -limit 25 -snap + filter_list sepolia all -limit 250 || return 1 + filter_list sepolia les -limit 25 -les-server || return 1 + filter_list sepolia snap -limit 25 -snap || return 1 # Holesky - filter_list holesky all -limit 250 - filter_list holesky snap -limit 25 -snap + filter_list holesky all -limit 250 || return 1 + filter_list holesky snap -limit 25 -snap || return 1 } sign_lists() { for D in *."${CRAWL_DNS_DOMAIN}"; do if [ -d "${D}" ]; then - echo "" | devp2p dns sign "${D}" "$CRAWL_DNS_SIGNING_KEY" + echo "" | devp2p dns sign "${D}" "$CRAWL_DNS_SIGNING_KEY" || return 1 fi done } @@ -92,7 +92,7 @@ sign_lists() { publish_dns_cloudflare() { for D in *."${CRAWL_DNS_DOMAIN}"; do if [ -d "${D}" ]; then - devp2p dns to-cloudflare -zoneid "$CLOUDFLARE_ZONE_ID" "${D}" + devp2p dns to-cloudflare -zoneid "$CLOUDFLARE_ZONE_ID" "${D}" || return 1 fi done } @@ -100,16 +100,16 @@ publish_dns_cloudflare() { publish_dns_route53() { for D in *."${CRAWL_DNS_DOMAIN}"; do if [ -d "${D}" ]; then - devp2p dns to-route53 -zone-id "$ROUTE53_ZONE_ID" "${D}" + devp2p dns to-route53 -zone-id "$ROUTE53_ZONE_ID" "${D}" || return 1 fi done } git_push_crawler_output() { if [ -n "$(git status --porcelain)" ]; then - git add all.json ./*."${CRAWL_DNS_DOMAIN}"/*.json - git commit --message "automatic update: crawl time $CRAWL_TIMEOUT" - git push origin "$CRAWL_GIT_BRANCH" + git add all.json ./*."${CRAWL_DNS_DOMAIN}"/*.json || return 1 + git commit --message "automatic update: crawl time $CRAWL_TIMEOUT" || return 1 + git push origin "$CRAWL_GIT_BRANCH" || return 1 fi } @@ -133,7 +133,7 @@ publish_influx_metrics() { init_prometheus_metrics() { go install -v github.com/projectdiscovery/simplehttpserver/cmd/simplehttpserver@v0.0.6 simplehttpserver -listen "${PROMETHEUS_METRICS_LISTEN}" -path "${prometheus_metrics_dir}" -silent & - publish_prometheus_metrics + publish_prometheus_metrics 0 # Init status with 0, which means it hasn't run yet } publish_prometheus_metrics() {