HARDER

scripts/update.sh

@@ -4,9 +4,15 @@
 # e.g.
 # ./scripts/update.sh
 
-# Cleanup sources.
+# Remove `www.` from sources.
 find ./sources -type f -name "*.txt" -exec sed -i 's/^www\.//i' {} \;
 
+# Remove duplicates from each source file.
+find ./sources -type f -name "*.txt" -exec bash -c '
+    cat -n "$0" | sort -uk2 | sort -nk1 | cut -f2- > "$0.temp";
+    mv "$0.temp" "$0";
+' {} \;
+
 # Combine all sources into `domains.txt`.
 find ./sources -type f -name "*.txt" -exec cat {} \; > sources.txt
 sed -i '/^#/d; s/#.*//; /^ *$/d' sources.txt

sources/eng/test2.txt

@@ -0,0 +1,9 @@
+# I'm testing!
+example.com # bad website
+www.example.com
+
+
+
+
+#
+EXAMPLEUZ.www.COM# FARTS