adblocker

merge_blocklists.sh

#!/usr/bin/env bash

lists=(
    'cad_servers.txt|https://hosts-file.net/.%5Cad_servers.txt'
    'justdomains.txt|http://mirror1.malwaredomains.com/files/justdomains'
    'immortal_domains.txt|http://mirror1.malwaredomains.com/files/immortal_domains.txt'
    'yous.txt|https://raw.githubusercontent.com/yous/YousList/master/hosts.txt'
    'RW_DOMBL.txt|https://ransomwaretracker.abuse.ch/downloads/RW_DOMBL.txt'
    'mdlcsv.txt|http://www.malwaredomainlist.com/hostslist/hosts.txt'
    'main-blacklist.txt|https://raw.githubusercontent.com/Dawsey21/Lists/master/main-blacklist.txt'
    'simple_malvertising.txt|https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt'
    'simple_malware.txt|https://s3.amazonaws.com/lists.disconnect.me/simple_malware.txt'
    'simple_tracking.txt|https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt'
    'simple_ad.txt|https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt'
    'o7Net.txt|https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.2o7Net/hosts'
    'risk.txt|https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Risk/hosts'
    'spam.txt|https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Spam/hosts'
    'Badd-Boyz.txt|https://raw.githubusercontent.com/mitchellkrogza/Badd-Boyz-Hosts/master/hosts'
    'sysctlhosts.txt|http://sysctl.org/cameleon/hosts'
    'KADhosts.txt|https://raw.githubusercontent.com/azet12/KADhosts/master/KADhosts.txt'
    'domain-list.txt|https://ssl.bblck.me/blacklists/domain-list.txt'
    'danhosts.txt|https://someonewhocares.org/hosts/hosts'
    'coinblocker.txt|https://zerodot1.gitlab.io/CoinBlockerLists/hosts'
    'StevenBlack.txt|https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts'
    'mpvs.txt|http://winhelp2002.mvps.org/hosts.txt'
    '1hosts.txt|http://1hosts.cf'
    'adaway.txt|https://adaway.org/hosts.txt'
    'pgl.txt|https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=0&mimetype=plaintext'
    'hoshsadiq.txt|https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt'
)

filter() {
    expand -t 1 "$1" >"$1".bak && mv "$1".bak "$1"
    sed -i "$1" \
        -e '/^#<localhost>/,/^#<\/localhost>/d' \
        -e 's;##\[href^=";;g' \
        -e 's;/"];;g' \
        -e 's;||;;g' \
        -e 's;\^;;g' \
        -e 's;^ #;;g' \
        -e 's;0.0.0.0.* ;;g' \
        -e 's;127.0.0.1.* ;;g' \
        -e 's;::1 localhost;;g' \
        -e 's;localhost;;g' \
        -e 's;.*Disconnect.*;;g' \
        -e 's;article;;g' \
        -e 's;en.wikipedia.org/wiki/List_of_shock_sites;;g' \
        -e 's;::1 IPv6;;g' \
        -e 's;::1;;g' \
        -e 's; IPv4;;g' \
        -e 's;http://;;g' \
        -e 's;https://;;g' \
        -e '/^.localdomain/,/^fe80%lo0 /d' \
        -e '/^ff00::0 ip6-localnet/,/^ff02::3 ip6-allhosts/d' \
        -e 's;0.0.0.0;;g' \
        -e '/^#/d' \
        -e 's/^ //' \
        -e '/^\s*#.*$/d' \
        -e '/^\s*$/d'
}

blockfile=hosts.block
tempdir=$(mktemp -d)
cd $tempdir

j=${#lists[@]}
echo -n "==> Downloading lists... "
tput sc

for i in ${lists[@]}; do
    ((j--))
    tput rc; tput ed
    echo -n "[$j]"

    file=$(echo $i |cut -d'|' -f1)
    url=$(echo $i |cut -d'|' -f2)

    curl -s "$url" -o "$file"
    filter $file
    cat $file >> ${blockfile}.tmp
done

echo
echo "==> Sorting..."
sort -f -u ${blockfile}.tmp |sed 's/^/0.0.0.0 /' >$blockfile

echo "==> Done."
echo " sudo mv $tempdir/$blockfile >  /etc/rescached/hosts.d/"
echo " rm -rf $tempdir"