Skip to content

Commit

Permalink
Optimize check_dead.sh
Browse files Browse the repository at this point in the history
  • Loading branch information
jarelllama authored Mar 29, 2024
1 parent 510b1e3 commit 822f455
Show file tree
Hide file tree
Showing 2 changed files with 26 additions and 26 deletions.
50 changes: 25 additions & 25 deletions functions/check_dead.sh
Original file line number Diff line number Diff line change
Expand Up @@ -15,29 +15,15 @@ function main {
for file in config/* data/*; do # Format files in the config and data directory
format_list "$file"
done
check_for_alive
check_subdomains
check_redundant
check_for_dead
check_dead
check_alive
cat dead_in_raw.tmp >> "$dead_domains_file" # Collate dead domains (skip alive check)
format_list "$dead_domains_file"
update_light_file
}

function check_for_alive {
sed 's/^/||/; s/$/^/' "$dead_domains_file" > formatted_dead_domains_file.tmp # Format dead domains file
dead-domains-linter -i formatted_dead_domains_file.tmp --export dead.tmp # Find dead domains in the dead domains file
alive_domains=$(comm -23 <(sort "$dead_domains_file") <(sort dead.tmp)) # Find resurrected domains in dead domains file (note dead domains file is unsorted)
[[ -z "$alive_domains" ]] && return # Return if no resurrected domains found
cp dead.tmp "$dead_domains_file" # Update dead domains file to exclude resurrected domains

# Strip away subdomains from alive domains since subdomains are not supposed to be in raw file
while read -r subdomain; do # Loop through common subdomains
alive_domains=$(printf "%s" "$alive_domains" | sed "s/^${subdomain}\.//" | sort -u)
done < "$subdomains_to_remove_file"
printf "%s\n" "$alive_domains" >> "$raw_file" # Add resurrected domains to raw file
format_list "$dead_domains_file" && format_list "$raw_file"
log_event "$alive_domains" "resurrected" "dead_domains_file"
}

function check_subdomains {
sed 's/^/||/; s/$/^/' "$subdomains_file" > formatted_subdomains_file.tmp # Format subdomains file
dead-domains-linter -i formatted_subdomains_file.tmp --export dead.tmp # Find and export dead domains with subdomains
Expand Down Expand Up @@ -77,16 +63,30 @@ function check_redundant {
log_event "$(<collated_dead_wildcards.tmp)" "dead" "wildcard"
}

function check_for_dead {
function check_dead {
comm -23 "$raw_file" <(sort "$root_domains_file" "$wildcards_file") | # Exclude wildcards and root domains of subdomains
sed 's/^/||/; s/$/^/' > formatted_raw_file.tmp # Format raw file
dead-domains-linter -i formatted_raw_file.tmp --export dead.tmp # Find and export dead domains
[[ ! -s dead.tmp ]] && return # Return if no dead domains found
dead-domains-linter -i formatted_raw_file.tmp --export dead_in_raw.tmp # Find and export dead domains
[[ ! -s dead_in_raw.tmp ]] && return # Return if no dead domains found
# Remove dead domains from raw file
comm -23 "$raw_file" dead.tmp > raw.tmp && mv raw.tmp "$raw_file"
cat dead.tmp >> "$dead_domains_file" # Collate dead domains
format_list "$dead_domains_file"
log_event "$(<dead.tmp)" "dead" "raw"
comm -23 "$raw_file" dead_in_raw.tmp > raw.tmp && mv raw.tmp "$raw_file"
log_event "$(<dead_in_raw.tmp)" "dead" "raw"
}

function check_alive {
sed 's/^/||/; s/$/^/' "$dead_domains_file" > formatted_dead_domains_file.tmp # Format dead domains file
dead-domains-linter -i formatted_dead_domains_file.tmp --export dead.tmp # Find dead domains in the dead domains file
alive_domains=$(comm -23 <(sort "$dead_domains_file") <(sort dead.tmp)) # Find resurrected domains in dead domains file (note dead domains file is unsorted)
[[ -z "$alive_domains" ]] && return # Return if no resurrected domains found
cp dead.tmp "$dead_domains_file" # Update dead domains file to exclude resurrected domains

# Strip away subdomains from alive domains since subdomains are not supposed to be in raw file
while read -r subdomain; do # Loop through common subdomains
alive_domains=$(printf "%s" "$alive_domains" | sed "s/^${subdomain}\.//" | sort -u)
done < "$subdomains_to_remove_file"
printf "%s\n" "$alive_domains" >> "$raw_file" # Add resurrected domains to raw file
format_list "$dead_domains_file" && format_list "$raw_file"
log_event "$alive_domains" "resurrected" "dead_domains_file"
}

function update_light_file {
Expand Down
2 changes: 1 addition & 1 deletion functions/check_parked.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ function main {
done
remove_parked_domains
add_unparked_domains
cat parked_domains.tmp >> "$parked_domains_file" # Add parked domains to parked domains file (skip unparked check)
cat parked_domains.tmp >> "$parked_domains_file" # Collate parked domains (skip unparked check)
format_list "$parked_domains_file"
update_light_file
}
Expand Down

0 comments on commit 822f455

Please sign in to comment.