Skip to content

Commit

Permalink
Update
Browse files Browse the repository at this point in the history
  • Loading branch information
@jarelllama
jarelllama authored Mar 29, 2024
1 parent 822f455 commit 43c5c7e
Show file tree
Hide file tree
Showing 6 changed files with 41 additions and 41 deletions.
9 changes: 7 additions & 2 deletions .github/workflows/build_deploy.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,10 @@ jobs:

deploy:
needs: validate
if: ${{ always() && needs.validate.result == 'success' }} # Deploy only if validate passes
if: ${{ success() || failure() }}
uses: ./.github/workflows/build_lists.yml

prune_logs:
prune-logs:
needs: deploy
if: ${{ success() || failure() }}
runs-on: ubuntu-latest
Expand All @@ -77,3 +77,8 @@ jobs:
git add .
git diff-index --quiet HEAD || git commit -m "Prune logs"
git push -q
update-readme:
needs: prune-logs
uses: ./.github/workflows/update_readme.yml
if: ${{ always() && needs.deploy.result == 'success' }} # Deploy only if deploy passes
2 changes: 1 addition & 1 deletion .github/workflows/update_readme.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ on:
workflow_dispatch:
workflow_call:
workflow_run:
workflows: [Build and deploy, Build lists]
workflows: Build lists
types:
- completed
push:
Expand Down
12 changes: 6 additions & 6 deletions functions/build_lists.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ function main {
build_wildcard_domains
}

function build_list {
function build_lists {
[[ -z "$comment" ]] && comment='#' # Set default comment to '#'
mkdir -p "lists/${directory}" # Create directory if not present

Expand Down Expand Up @@ -43,27 +43,27 @@ function format_list {

function build_adblock {
syntax='Adblock Plus' && directory='adblock' && comment='!' && before='||' && after='^'
build_list
build_lists
}

function build_dnsmasq {
syntax='Dnsmasq' && directory='dnsmasq' && comment='' && before='local=/' && after='/'
build_list
build_lists
}

function build_unbound {
syntax='Unbound' && directory='unbound' && comment='' && before='local-zone: "' && after='." always_nxdomain'
build_list
build_lists
}

function build_wildcard_asterisk {
syntax='Wildcard Asterisk' && directory='wildcard_asterisk' && comment='' && before='*.' && after=''
build_list
build_lists
}

function build_wildcard_domains {
syntax='Wildcard Domains' && directory='wildcard_domains' && comment='' && before='' && after=''
build_list
build_lists
}

main
36 changes: 18 additions & 18 deletions functions/check_parked.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,13 @@ function remove_parked_domains {

# Split into 12 equal files
split -d -l $(($(wc -l < "$raw_file")/12)) "$raw_file"
check_for_parked "x00" & check_for_parked "x01" &
check_for_parked "x02" & check_for_parked "x03" &
check_for_parked "x04" & check_for_parked "x05" &
check_for_parked "x06" & check_for_parked "x07" &
check_for_parked "x08" & check_for_parked "x09" &
check_for_parked "x10" & check_for_parked "x11" &
check_for_parked "x12" & check_for_parked "x13"
check_parked "x00" & check_parked "x01" &
check_parked "x02" & check_parked "x03" &
check_parked "x04" & check_parked "x05" &
check_parked "x06" & check_parked "x07" &
check_parked "x08" & check_parked "x09" &
check_parked "x10" & check_parked "x11" &
check_parked "x12" & check_parked "x13"
wait

[[ ! -s parked_domains.tmp ]] && return
Expand All @@ -47,13 +47,13 @@ function add_unparked_domains {

# Split into 12 equal files
split -d -l $(($(wc -l < "$parked_domains_file")/12)) "$parked_domains_file"
check_for_unparked "x00" & check_for_unparked "x01" &
check_for_unparked "x02" & check_for_unparked "x03" &
check_for_unparked "x04" & check_for_unparked "x05" &
check_for_unparked "x06" & check_for_unparked "x07" &
check_for_unparked "x08" & check_for_unparked "x09" &
check_for_unparked "x10" & check_for_unparked "x11" &
check_for_unparked "x12" & check_for_unparked "x13"
check_unparked "x00" & check_unparked "x01" &
check_unparked "x02" & check_unparked "x03" &
check_unparked "x04" & check_unparked "x05" &
check_unparked "x06" & check_unparked "x07" &
check_unparked "x08" & check_unparked "x09" &
check_unparked "x10" & check_unparked "x11" &
check_unparked "x12" & check_unparked "x13"
wait

[[ ! -s unparked_domains.tmp ]] && return
Expand All @@ -67,13 +67,13 @@ function add_unparked_domains {
find . -maxdepth 1 -type f -name "x??" -delete # Reset split files before next run
}

function check_for_parked {
function check_parked {
[[ ! -f "$1" ]] && return # Return if split file not found
[[ "$1" == 'x00' ]] && { track=true; count=0; } || track=false # Track progress for first split file
while read -r domain; do
((count++))
# Check for parked message in site's HTML
if grep -qiFf "$parked_terms_file" <<< "$(curl -sL --max-time 2 "http://${domain}/" | tr -d '\0')"; then
if grep -qiaFf "$parked_terms_file" <<< "$(curl -sL --max-time 2 "http://${domain}/")"; then
printf "Parked: %s\n" "$domain"
printf "%s\n" "$domain" >> "parked_domains_${1}.tmp"
fi
Expand All @@ -85,13 +85,13 @@ function check_for_parked {
[[ -f "parked_domains_${1}.tmp" ]] && cat "parked_domains_${1}.tmp" >> parked_domains.tmp
}

function check_for_unparked {
function check_unparked {
[[ ! -f "$1" ]] && return # Return if split file not found
[[ "$1" == 'x00' ]] && { track=true; count=0; } || track=false # Track progress for first split file
while read -r domain; do
((count++))
# Check for parked message in site's HTML
if ! grep -qiFf "$parked_terms_file" <<< "$(curl -sL --max-time 5 "http://${domain}/" | tr -d '\0')"; then
if ! grep -qiaFf "$parked_terms_file" <<< "$(curl -sL --max-time 5 "http://${domain}/")"; then
printf "Unparked: %s\n" "$domain"
printf "%s\n" "$domain" >> "unparked_domains_${1}.tmp"
fi
Expand Down
9 changes: 4 additions & 5 deletions functions/retrieve_domains.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,8 @@ function main {
}

function source {
printf "\n"
# Check for existing pending domains file
[[ -d data/pending ]] && { use_pending=true; printf "Using existing lists of retrieved domains.\n\n"; }
[[ -d data/pending ]] && { use_pending=true; printf "\nUsing existing lists of retrieved domains.\n"; }
[[ -f data/pending/domains_manual.tmp ]] && source_manual # Retrieve manually added domains
mkdir -p data/pending
source_aa419
Expand Down Expand Up @@ -196,6 +195,7 @@ function process_source {
[[ -z "$rate_limited" ]] && rate_limited=false
[[ -z "$ignore_from_light" ]] && ignore_from_light=false

printf "\n"
[[ ! -f "$domains_file" ]] && return # Return if domains file does not exist
! grep -q '[[:alnum:]]' "$domains_file" && { log_source; return; } # Skip to next source if no results retrieved

Expand Down Expand Up @@ -285,7 +285,7 @@ function process_source {
fi

total_whitelisted_count=$((whitelisted_count + whitelisted_tld_count)) # Calculate sum of whitelisted domains
filtered_count=$(tr -s '\n' <<< "$pending_domains" | wc -w) # Count number of domains after filtering
filtered_count=$(printf "%s" "$pending_domains" | sed '/^$/d' | wc -w) # Count number of domains after filtering
printf "%s\n" "$pending_domains" >> retrieved_domains.tmp # Collate filtered domains
[[ "$ignore_from_light" != true ]] && printf "%s\n" "$pending_domains" >> retrieved_light_domains.tmp # Collate filtered domains from light sources
log_source
Expand Down Expand Up @@ -325,8 +325,7 @@ function build {
format_list "$raw_file"
log_event "$(<retrieved_domains.tmp)" "new_domain" "retrieval"
count_after=$(wc -l < "$raw_file")
count_difference=$((count_after - count_before))
printf "\nAdded new domains to blocklist.\nBefore: %s Added: %s After: %s\n" "$count_before" "$count_difference" "$count_after"
printf "\nAdded new domains to blocklist.\nBefore: %s Added: %s After: %s\n" "$count_before" "$((count_after - count_before))" "$count_after"

# Mark sources as saved in the source log file
rows=$(sed 's/,no/,yes/' <(grep -F "$time_format" "$source_log")) # Record that the domains were saved into the raw file
Expand Down
14 changes: 5 additions & 9 deletions functions/test_functions.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ function shellcheck {
fi

printf "\nScripts checked (%s):\n%s\n" "$(wc -l <<< "$scripts")" "$scripts"
check_error
[[ "$error" == true ]] && { printf "\n"; exit 1; } # Exit with error if test failed
}

function test_retrieval_validate {
Expand Down Expand Up @@ -206,7 +206,7 @@ function test_retrieval_validate {

[[ "$error" != true ]] && printf "Test completed. No errors found.\n\n"
[[ "$log_error" != true ]] && printf "Log:\n%s\n" "$(<$domain_log)"
check_error
[[ "$error" == true ]] && { printf "\n"; exit 1; } # Exit with error if test failed
}

function test_toplist_check {
Expand All @@ -220,7 +220,7 @@ function test_toplist_check {
[[ ! -d data/pending ]] && { printf "! Pending directory is missing.\n"; error=true; } # Check pending directory
check_log
[[ "$error" != true ]] && printf "Test completed. No errors found.\n\n"
check_error
[[ "$error" == true ]] && { printf "\n"; exit 1; } # Exit with error if test failed
}

function test_dead_check {
Expand Down Expand Up @@ -292,7 +292,7 @@ function test_dead_check {
[[ "$error" != true ]] && printf "Test completed. No errors found.\n\n" ||
printf "The dead-domains-linter may have false positives. Rerun the job to confirm.\n\n"
[[ "$log_error" != true ]] && printf "Log:\n%s\n" "$(<$domain_log)"
check_error
[[ "$error" == true ]] && { printf "\n"; exit 1; } # Exit with error if test failed
}

function test_parked_check {
Expand Down Expand Up @@ -340,7 +340,7 @@ function test_parked_check {
check_output "$parked_domains_file" "out_parked.txt" "Parked domains" # Check parked domains file
check_log # Check log file
[[ "$error" != true ]] && printf "Test completed. No errors found.\n\n"
check_error
[[ "$error" == true ]] && { printf "\n"; exit 1; } # Exit with error if test failed
}

function prep_output {
Expand Down Expand Up @@ -379,8 +379,4 @@ function check_log {
error=true
}

function check_error {
[[ "$error" == true ]] && { printf "\n"; exit 1; } # Exit with error if test failed
}

main "$1"

0 comments on commit 43c5c7e

Please sign in to comment.