nrd-list-downloader.sh

#!/usr/bin/env bash

# nrd-list-downloader
# https://github.com/PeterDaveHello/nrd-list-downloader
# Copyright (C) 2022 ~ Peter Dave Hello
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# ColorEchoForShell #
# https://github.com/PeterDaveHello/ColorEchoForShell
# Copyright (C) 2015 ~ Peter Dave Hello

function echo.Red() {
  echo -e "\\033[31m$*\\033[m"
}

function echo.Green() {
  echo -e "\\033[32m$*\\033[m"
}

function echo.Cyan() {
  echo -e "\\033[36m$*\\033[m"
}
# ColorEchoForShell End #

function error() {
    echo.Red >&2 "$@"
    exit 1
}

for cmd in mkdir wc base64 curl cat zcat mktemp date tr realpath dirname; do
    if ! command -v "$cmd" > /dev/null 2>&1; then
        error "command: $cmd not found!"
    fi
done

set -e

DIR="$(dirname "$(realpath "${BASH_SOURCE[0]}")")"
DAY_RANGE="${DAY_RANGE:-7}"
DAILY_DIR="${DAILY_DIR:-daily}"
TEMP_FILE="$(mktemp -p "$DIR" --suffix=nrd)"

PAID_WHOISDS_USERNAME="${PAID_WHOISDS_USERNAME:-}"
PAID_WHOISDS_PASSWORD="${PAID_WHOISDS_PASSWORD:-}"

BASE_URL_FREE="https://whoisds.com/whois-database/newly-registered-domains"
BASE_URL_PAID="https://whoisds.com/your-download/direct-download_file/${PAID_WHOISDS_USERNAME}/${PAID_WHOISDS_PASSWORD}"

COMMENT="# NRD list generated by nrd-list-downloader, on $(date '+%Y-%m-%d'), data source: WhoisDS.com"

cd "$DIR"

echo.Green "You are using nrd-list-downloader to download NRD(Newly Registered Domain) list ..."

echo.Cyan "NRD list of the last $DAY_RANGE days will be downloaded."

function insert_into_temp_file() { echo "$*" >> "$TEMP_FILE"; }

insert_into_temp_file "$COMMENT"

function download() {

    local TYPE="${1:-free}"

    local TARGET_FILE="nrd-${DAY_RANGE}days-${TYPE}.txt"
    local DOWNLOAD_DIR="${DAILY_DIR}/${TYPE}"
    mkdir -p "$DOWNLOAD_DIR"

    echo
    echo.Cyan "Downloading $TYPE NRD list ..."

    if [ "free" = "$TYPE" ] && [ "$DAY_RANGE" -gt "10" ]; then
        echo.Red "Warning! Free NRD list before more than 10 days might be removed from WhoisDS.com already, the download may failed."
    fi
    if [ "paid" = "$TYPE" ] && [ "$DAY_RANGE" -gt "30" ]; then
        echo.Red "Warning! Paid NRD list before more than 30 days might be removed from WhoisDS.com already, the download may failed."
    fi

    local i="$DAY_RANGE"
    while [ "$i" -gt "0" ]; do
        local DATE FILE URL FREE_URL_INFIX
        DATE="$(date -u --date "$i days ago" '+%Y-%m-%d')"
        FILE="${DOWNLOAD_DIR}/${DATE}"
        if [ -s "$FILE" ] && [ "$(grep -vc '^$' "$FILE")" -ge "1" ] ; then
            echo.Cyan "$FILE existed with $(grep -vc '^$' "$FILE") domains, skip the download and decompress process ..."
        else
            printf "%s" "Download and decompress $DATE data ... "
            if [ "paid" = "$TYPE" ]; then
                URL="${BASE_URL_PAID}/${DATE}.zip/ddu"
            else
                FREE_URL_INFIX="$(echo "${DATE}.zip" | base64)"
                URL="${BASE_URL_FREE}/${FREE_URL_INFIX:0:-1}/nrd"
            fi
            curl -sSLo- "$URL" | zcat | tr -d '\015' >> "$FILE"
            echo "" >> "$FILE"
            echo.Cyan "$(grep -vc '^$' "$FILE") domains found."
        fi
        insert_into_temp_file "# ${DATE} NRD start"
        cat "$FILE" >> "$TEMP_FILE"
        insert_into_temp_file "# ${DATE} NRD end"
        i="$((i - 1))"
    done

    insert_into_temp_file "$COMMENT"

    chmod +r "$TEMP_FILE"
    mv "$TEMP_FILE" "$TARGET_FILE"

    echo.Green "NRD list for the last $DAY_RANGE days saved to $TARGET_FILE, $(grep -cvE '^(#|$)' "$TARGET_FILE") domains found."
    echo
}

download free

if [ -n "$PAID_WHOISDS_USERNAME" ] && [ -n "$PAID_WHOISDS_PASSWORD" ]; then
    echo.Green "WhoisDS paid account found! Will try to download paid premium NRD package with it."
    download paid
fi