Write NS of whitelists to /etc/hosts

package.json

@@ -1,6 +1,6 @@
 {
   "name": "domcloud-bridge",
-  "version": "0.43.2",
+  "version": "0.44.0",
   "description": "Deployment runner for DOM Cloud",
   "main": "app.js",
   "engines": {

src/whitelist/.gitignore

@@ -0,0 +1 @@
+*.txt

src/whitelist/refresh.sh

@@ -1,29 +1,32 @@
 #!/bin/bash
+set -e 
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 
+bash "$SCRIPT_DIR/resolve.sh"
+
 ### Create Ipset
 ipset -! create whitelist hash:ip
 ipset -! create whitelist-v6 hash:ip family inet6
-### Clear Ipset, not gonna use it because DNS IPs often changing
-# ipset flush whitelist
-# ipset flush whitelist-v6
+### Clear Ipset
+ipset flush whitelist
+ipset flush whitelist-v6
+
 while read p; do
-  if [[ $p != "#"* ]];
+  if [[ $p != "" ]];
   then
-    FFI=`dig +short A $(echo $p | xargs) | grep -v '\.$'`
-    while read -r q; do
-      if [[ $q != "" ]];
-      then
-        ipset -! add whitelist $q
-      fi
-    done < <(echo $FFI| sed 's/ /\n/g')
-    FFI6=`dig +short AAAA $(echo $p | xargs) | grep -v '\.$'`
-    while read -r q; do
-      if [[ $q != "" ]];
-      then
-        ipset -! add whitelist-v6 $q
-      fi
-    done < <(echo $FFI6| sed 's/ /\n/g')
-  fi
-done <"$SCRIPT_DIR/sites.conf"
+    ipset -! add whitelist $q
+done <"$SCRIPT_DIR/ipv4_addresses.txt"
+
+while read p; do
+  if [[ $p != "" ]];
+  then
+    ipset -! add whitelist_v6 $q
+done <"$SCRIPT_DIR/ipv6_addresses.txt"
+
+if [ ! -f "$SCRIPT_DIR/hosts.txt" ]; then
+    cat /etc/hosts > "$SCRIPT_DIR/hosts.txt"
+fi
+
+cat "$SCRIPT_DIR/hosts.txt" > /etc/hosts
+cat "$SCRIPT_DIR/host_addresses.txt" >> /etc/hosts

src/whitelist/resolve.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+IPV4_ADDRESSES=""
+IPV6_ADDRESSES=""
+HOST_ADDRESSES=""
+
+for RECORD_TYPE in A AAAA; do
+  while read -r p; do
+    if [[ $p != "#"* ]]; then
+      printf '\r%s Fetching NS %s of %s' "$(tput el)" $RECORD_TYPE $p
+      FFI=$(dig +short $RECORD_TYPE $(echo "$p" | xargs) | grep -v '\.$' | tail -n1)
+      while read -r q; do
+        if [[ $q != "" ]]; then
+          HOST_ADDRESSES+="$q $p"$'\n'
+          if [[ $RECORD_TYPE == "A" ]]; then
+            IPV4_ADDRESSES+="$q"$'\n'
+          else
+            IPV6_ADDRESSES+="$q"$'\n'
+          fi
+        fi
+      done <<< "$FFI"
+    fi
+  done <"$SCRIPT_DIR/sites.conf"
+done 
+
+printf '\n'
+
+echo "$IPV4_ADDRESSES" > ipv4_adresses.txt
+echo "$IPV6_ADDRESSES" > ipv6_adresses.txt
+echo "$HOST_ADDRESSES" > host_adresses.txt

src/whitelist/sites.conf

@@ -6,11 +6,6 @@
 # 2. Used by a lot of people
 # 3. Easily handle million hits
 #
-# Yum repository
-mirror.centos.org
-mirrors.fedoraproject.org
-mirror.rockylinux.org
-ftp.redhat.com
 # GitHub
 github.com
 api.github.com
@@ -71,7 +66,7 @@ public-api.wordpress.com
 connect.woocommerce.com
 websitedemos.net
 wordpress.org
-# SEO / Recaptcha / Google APIs
+# Google APIs
 google.com
 www.google.com
 www.googleapis.com