Skip to content

Commit

Permalink
Fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
jarelllama authored Apr 3, 2024
1 parent 4f0ab54 commit 3bf60ac
Show file tree
Hide file tree
Showing 4 changed files with 14 additions and 10 deletions.
3 changes: 2 additions & 1 deletion functions/check_dead.sh
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,8 @@ find_dead() {
# $3: source
log_event() {
[[ -z "$1" ]] && return # Return if no domains in variable
printf "%s\n" "$1" | awk -v type="$2" -v source="$3" -v time="$(date -u +"%H:%M:%S %d-%m-%y")" \
local source="$3"
printf "%s\n" "$1" | awk -v type="$2" -v source="$source" -v time="$(date -u +"%H:%M:%S %d-%m-%y")" \
'{print time "," type "," $0 "," source}' >> "$DOMAIN_LOG"
}

Expand Down
3 changes: 2 additions & 1 deletion functions/check_parked.sh
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,8 @@ find_parked() {
# $3: source
log_event() {
[[ -z "$1" ]] && return # Return if no domains in variable
printf "%s\n" "$1" | awk -v type="$2" -v source="$3" -v time="$(date -u +"%H:%M:%S %d-%m-%y")" \
local source="$3"
printf "%s\n" "$1" | awk -v type="$2" -v source="$source" -v time="$(date -u +"%H:%M:%S %d-%m-%y")" \
'{print time "," type "," $0 "," source}' >> "$DOMAIN_LOG"
}

Expand Down
3 changes: 2 additions & 1 deletion functions/retrieve_domains.sh
Original file line number Diff line number Diff line change
Expand Up @@ -278,7 +278,8 @@ ${query_count:-0},${rate_limited:-false},no" >> "$SOURCE_LOG"
# $3: source
log_event() {
[[ -z "$1" ]] && return # Return if no domains in variable
printf "%s\n" "$1" | awk -v type="$2" -v source="$3" -v time="$(date -u +"%H:%M:%S %d-%m-%y")" \
[[ -n "$3" ]] && local source="$3" # Use specific source if passed
printf "%s\n" "$1" | awk -v type="$2" -v source="$source" -v time="$(date -u +"%H:%M:%S %d-%m-%y")" \
'{print time "," type "," $0 "," source}' >> "$DOMAIN_LOG"
}

Expand Down
15 changes: 8 additions & 7 deletions functions/validate_raw.sh
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ validate_raw() {
printf "%s\n" "$domains_with_subdomains" | sed "s/^${subdomain}\.//" >> root_domains.tmp

awk '{print $0 " (subdomain)"}' <<< "$domains_with_subdomains" >> filter_log.tmp
log_event "$domains_with_subdomains" subdomain raw
log_event "$domains_with_subdomains" subdomain
done < "$SUBDOMAINS_TO_REMOVE"
format_file subdomains.tmp
format_file root_domains.tmp
Expand All @@ -55,7 +55,7 @@ validate_raw() {
if (( whitelisted_count > 0 )); then
domains="$(comm -23 <(printf "%s" "$domains") <(printf "%s" "$whitelisted_domains"))"
awk '{print $0 " (whitelisted)"}' <<< "$whitelisted_domains" >> filter_log.tmp
log_event "$whitelisted_domains" whitelist raw
log_event "$whitelisted_domains" whitelist
fi

# Remove domains that have whitelisted TLDs
Expand All @@ -64,7 +64,7 @@ validate_raw() {
if (( whitelisted_tld_count > 0 )); then
domains="$(comm -23 <(printf "%s" "$domains") <(printf "%s" "$whitelisted_tld_domains"))"
awk '{print $0 " (whitelisted TLD)"}' <<< "$whitelisted_tld_domains" >> filter_log.tmp
log_event "$whitelisted_tld_domains" tld raw
log_event "$whitelisted_tld_domains" tld
fi

# Remove invalid entries and IP addresses. Punycode TLDs (.xn--*) are allowed
Expand All @@ -73,7 +73,7 @@ validate_raw() {
if (( invalid_entries_count > 0 )); then
domains="$(comm -23 <(printf "%s" "$domains") <(printf "%s" "$invalid_entries"))"
awk '{print $0 " (invalid)"}' <<< "$invalid_entries" >> filter_log.tmp
log_event "$invalid_entries" invalid raw
log_event "$invalid_entries" invalid
fi

# Remove redundant domains
Expand All @@ -96,7 +96,7 @@ validate_raw() {
printf "%s\n" "$domain" >> wildcards.tmp

awk '{print $0 " (redundant)"}' <<< "$redundant_domains" >> filter_log.tmp
log_event "$redundant_domains" redundant raw
log_event "$redundant_domains" redundant
done <<< "$domains"
format_file redundant_domains.tmp
format_file wildcards.tmp
Expand All @@ -108,7 +108,7 @@ validate_raw() {
if (( toplist_count > 0 )); then
awk '{print $0 " (toplist) - \033[1;31mmanual removal required\033[0m"}' \
<<< "$domains_in_toplist" >> filter_log.tmp
log_event "$domains_in_toplist" toplist raw
log_event "$domains_in_toplist" toplist
fi

# Exit if no filtering done
Expand Down Expand Up @@ -166,7 +166,8 @@ validate_raw() {
# $3: source
log_event() {
[[ -z "$1" ]] && return # Return if no domains in variable
printf "%s\n" "$1" | awk -v type="$2" -v source="$3" -v time="$(date -u +"%H:%M:%S %d-%m-%y")" \
local source='raw'
printf "%s\n" "$1" | awk -v type="$2" -v source="$source" -v time="$(date -u +"%H:%M:%S %d-%m-%y")" \
'{print time "," type "," $0 "," source}' >> "$DOMAIN_LOG"
}

Expand Down

0 comments on commit 3bf60ac

Please sign in to comment.