whitelists collection script for pihole and the likes

whitelistcat.sh

#!/bin/bash

wget -qO - --limit-rate=1500k https://dbl.oisd.nl/whitelist2.txt > /tmp/white
sleep 2
wget -qO - --limit-rate=1500k https://www.technoy.de/lists/whitelist.txt >> /tmp/white
sleep 2
wget -qO - --limit-rate=1500k https://raw.githubusercontent.com/raghavdua1995/DNSlock-PiHole-whitelist/master/whitelist.list >> /tmp/white
sleep 2
wget -qO - --limit-rate=1500k https://raw.githubusercontent.com/anudeepND/whitelist/master/domains/referral-sites.txt >> /tmp/white
sleep 2
wget -qO - --limit-rate=1500k https://raw.githubusercontent.com/anudeepND/whitelist/master/domains/whitelist.txt >> /tmp/white
sleep 2
wget -qO - --limit-rate=1500k https://raw.githubusercontent.com/anudeepND/whitelist/master/domains/optional-list.txt >> /tmp/white
sleep 2
wget -qO - --limit-rate=1500k https://raw.githubusercontent.com/jumpsmm7/dnscrypt-lists/master/whitelists/amazon.txt >> /tmp/white
sleep 2
wget -qO - --limit-rate=1500k https://raw.githubusercontent.com/jumpsmm7/dnscrypt-lists/master/whitelists/google.txt  >> /tmp/white
sleep 2
wget -qO - --limit-rate=1500k https://raw.githubusercontent.com/jumpsmm7/dnscrypt-lists/master/whitelists/microsoft.txt >> /tmp/white
sleep 2
wget -qO - --limit-rate=1500k https://raw.githubusercontent.com/jumpsmm7/dnscrypt-lists/master/whitelists/ocsp.txt >> /tmp/white
sleep 2
wget -qO - --limit-rate=1500k https://raw.githubusercontent.com/jumpsmm7/dnscrypt-lists/master/whitelists/social.txt >> /tmp/white
sleep 2
wget -qO - --limit-rate=1500k https://raw.githubusercontent.com/mmotti/pihole-regex/master/whitelist.list >> /tmp/white
sleep 2

# grab my own custom list and add it
cat /w/jult/whitey >> /tmp/white
sleep 2

# We're on a linux device here, so correct linebreaks
dos2unix -q -n /tmp/white /tmp/wlistnix
sleep 2

# gotta love uniq (sorting and removing doubles)
sort -u /tmp/wlistnix > /tmp/wlisort
sleep 1

# remove invalid start of lines
sed '/^[a-zA-Z0-9]/!d' < /tmp/wlisort > /tmp/whitee
# remove invalid hostnames
sed -r '/[.].*[a-zA-Z0-9][a-zA-Z0-9-]+([.][a-zA-Z]{2,15})?$/!d' < /tmp/whitee > /tmp/whiteee

cp -f /tmp/whiteee /w/jult/white.txt
mv -f /tmp/whiteee /w/jult/whitelist.txt

# Let's make sure next run is a clean one
rm -rf /tmp/white*
rm -rf /tmp/wl*

exit 0