Blocking evil servers via dnsmasq

hostsblock.sh

#!/usr/bin/env zsh

# MODIFY SETTINGS VIA THE CONFIGURATION FILES IN
# conf/

# eval PATH=/opt/usr/sbin:/opt/etc/init.d:$PATH

# CHECK FOR NEEDED AND OPTIONAL UTILITIES AND PERMISSIONS
# if [ `whoami` != "root" ]; then
#     echo "Insufficient permissions. Run as root."
#     exit 1
# fi

for dep in curl grep sed tr cut; do
    if which "$dep" &>/dev/null; then
        true
    else
        if [ "$dep" == "tr" ]; then
            echo "coreutils is not installed or not in your PATH. Please remedy. Exiting."
        else
            echo "Utility $dep not installed or not in your PATH. Please remedy. Exiting."
        fi
        exit 1
    fi
done

if which unzip &>/dev/null; then
    zip="1"
else
    echo "Dearchiver unzip not found. URLs which use this format will be skipped."
    zip="0"
fi
if which 7za &>/dev/null; then
    zip7="1"
else
    echo "Dearchiver 7za not found. URLs which use this format will be skipped."
    zip7="0"
fi

# DEFAULT SETTINGS
_dir="${PWD}"
tmpdir="tmp"
hostsfile="hosts"
redirecturl="127.0.0.1"
endfile="$_dir/dnsmasq.hosts.conf"
# service dnsmasq restart ?
postprocess(){
    :
} # function cannot be empty. ':' is null directive
blocklists="http://support.it-mate.co.uk/downloads/HOSTS.txt"
logfile="/Users/rrj/Library/Logs/hostsblock.log"
blacklist="$_dir/conf/black.list"
whitelist="$_dir/conf/white.list"
hostshead="0"
cachedir="cache"
redirects="0"

# CHECK TO SEE IF WE ARE LOGGING THIS
if [ "$logfile" != "0" ]; then
    exec > "$logfile" 2>&1
fi

printf "\nHostsblock started at `date +'%x %T'`"
printf "\n"

# READ CONFIGURATION FILE.
if [ -f  /Users/rrj/Documents/Shell/hostsblock/conf/hostsblock.conf ]; then
    . conf/hostsblock.conf
else
    printf "Config file conf/hostsblock.conf not found. Using defaults."
fi

# CREATE CACHE DIRECTORY IF NOT ALREADY EXISTANT
[ -d "$cachedir" ] || mkdir -p "$cachedir"

# DOWNLOAD BLOCKLISTS
changed=1
printf "\nChecking blocklists for updates...\n"

if [ -f conf/blocklists.csv ]; then
    OLD_IFS=$IFS
    IFS=','
    while read switch url ; do
        if [ "$switch" == "1" ]; then
            printf "   `echo $url | tr -d '%'`..."
            outfile=`echo $url | sed 's|http:\/\/||g' | tr '/%&+?=' '.'`
            [ -f "$cachedir"/"$outfile" ] && old_ls=`ls -l "$cachedir"/"$outfile"`
            if curl --compressed --connect-timeout 60 -sz "$cachedir"/"$outfile" "$url" -o "$cachedir"/"$outfile"; then
                new_ls=`ls -l "$cachedir"/"$outfile"`
                if [ "$old_ls" != "$new_ls" ]; then
                    changed=1
            printf "UPDATED\n"
        else
            printf "no changes\n"
        fi
    else
        printf "FAILED\nScript exiting @ `date +'%x %T'`"
        exit 1
    fi
        else
            continue
        fi
    done < /Users/rrj/Documents/Shell/hostsblock/conf/blocklists.csv
    IFS=$OLD_IFS
else
    printf "Config file conf/blocklists.csv not found."
fi

# IF THERE ARE CHANGES...
 if [ "$changed" != "0" ]; then
   printf "\nDONE. Changes found."

    # CREATE TMPDIR
    [ -d "$tmpdir"/hosts.d ] || mkdir -p "$tmpdir"/hosts.d

    # BACK UP FINAL HOSTSFILE IF IT EXISTS
    if [ -f $endfile ]
    then
        printf "\nBacking up $endfile to $endfile.old..."
        cp "$endfile" "$endfile".old && printf "done" || printf "FAILED"
        rm "$endfile"
    else
        touch "$endfile"
    fi

    # EXTRACT CACHED FILES TO HOSTS.BLOCK.D
    printf "\n\nExtracting and preparing cached files to working directory..."
    n=1
    if [ -f /Users/rrj/Documents/Shell/hostsblock/conf/blocklists.csv ]; then
        OLD_IFS=$IFS
        IFS=','
        while read switch url ; do
            if [ "$switch" == "1" ]; then
        FILE=`echo $url | sed "s|http:\/\/||g" | tr '/%&=?' '.'`
        printf "\n    `basename $FILE | tr -d '\%'`..."
        case "$FILE" in
            *".zip")
                if [ $zip == "1" ]; then
                    mkdir "$tmpdir"/tmp
                    cp "$cachedir"/"$FILE" "$tmpdir"/tmp
                    cd "$tmpdir"/tmp
                    printf "extracting..."
                    unzip -jq "$FILE" &>/dev/null && printf "extracted..." || printf "FAILED"
                    grep -rIh -- "^[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}" ./* > "$tmpdir"/hosts.d/hosts.block.$n
                    cd "$tmpdir"
                    rm -r "$tmpdir"/tmp
                    printf "prepared"
                else
                    printf "unzip not found. Skipping"
                fi
            ;;
            *".7z")
                if [ $zip7 == "1" ]; then
                    mkdir "$tmpdir"/tmp
                    cp "$cachedir"/"$FILE" "$tmpdir"/tmp
                    cd "$tmpdir"/tmp
                    printf "extracting..."
                    7za e "$FILE" &>/dev/null && printf "extracted..." || printf "FAILED"
                    grep -rIh -- "^[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}" ./* > "$tmpdir"/hosts.d/hosts.block.$n
                    cd "$tmpdir"
                    rm -r "$tmpdir"/tmp
                    printf "prepared"
                else
                    printf "7za not found. Skipping"
                fi
            ;;
            *)
                cp "$cachedir"/"$FILE" "$tmpdir"/hosts.d/hosts.block.$n && printf "prepared" || printf "FAILED"
            ;;
        esac
        let "n+=1"
            else
                continue
            fi
        done < /Users/rrj/Documents/Shell/hostsblock/conf/blocklists.csv
        IFS=$OLD_IFS
    else
        printf "Config file /Users/rrj/Documents/Shell/hostsblock/conf/blocklists.csv not found."
    fi

    # INCLUDE LOCAL BLACKLIST FILE
    printf "\n    Local blacklist..."
    cat "$blacklist" |\
    sed "s|^|$redirecturl |g" >> "$tmpdir"/hosts.d/hosts.block.0 && echo "prepared" || echo "FAILED"

    # GENERATE WHITELIST SED SCRIPT
    printf "\n    Local whitelist..."
    cat "$whitelist" |\
    sed -e 's/.*/\/&\/d/' -e 's/\./\\./g' >> "$tmpdir"/whitelist.sed && printf "prepared" || printf "FAILED"

    # DETERMINE THE REDIRECT URL NOT BEING USED
    if [ "$redirecturl" == "127.0.0.1" ]; then
        notredirect="0.0.0.0"
    else
        notredirect="127.0.0.1"
    fi
    printf "\n redirecturl="
    printf "$redirecturl"

    # PROCESS BLOCKLIST ENTRIES INTO TARGET FILE
    if [ "$hostshead" != "0" ]; then
        cp -f "$hostshead" "$hostsfile"
    else
        touch "$hostsfile"
    fi

    printf "\nDONE.\nNow processing files..."
    # DETERMINE WHETHER TO INCLUDE REDIRECTIONS
    if [ "$redirects" == "1" ]; then
        grep_eval='grep -Ih -- "^[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}" "$tmpdir"/hosts.d/*'
    else
        grep_eval='grep -IhE -- "^127\.0\.0\.1|^0\.0\.0\.0" "$tmpdir"/hosts.d/*'
    fi

    # PROCESS AND WRITE TO FILE
    eval $grep_eval | sed -e 's/[[:space:]][[:space:]]*/ /g' -e "s/\#.*//g" -e "s/[[:space:]]$//g" -e \
    "s/$notredirect/$redirecturl/g" | sort -u | sed -f "$tmpdir"/whitelist.sed >> "$hostsfile" && printf "done\n"

    # APPEND BLACKLIST ENTRIES
    printf "\nAppending blacklist entries..."
    cat "$blacklist" |\
        sed "s|^|$redirecturl |g" >> "$hostsfile" && printf "done\n" || printf "FAILED\n"

    # REPORT COUNT OF MODIFIED OR BLOCKED URLS
    for addr in `grep -Ih -- "^[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}" "$hostsfile" | cut -d" " -f1 | sort -u |\
    tr '\n' ' '`; do
        number=`grep -c -- "^$addr" "$hostsfile"`
        if [ "$addr" == "$redirecturl" ]; then
            printf "\n$number urls blocked"
        else
            printf "\n$number urls redirected to $addr"
        fi
    done

    #Выбираем только нужные строки из чёрного списка
    sed -i '' -e '/^[0-9A-Za-z]/!d' "$hostsfile"
    sed -i '' -e '/%/d' "$hostsfile"
    sed -i '' -e 's/[[:cntrl:][:blank:]]//g' "$hostsfile"
    sed -i '' -e 's/^[ \t]*//;s/[ \t]*$//' "$hostsfile"

    # dnsmasq, чистим, оптимизируем
    sed -i '' -e 's/[[:space:]]*\[.*$//'  "$hostsfile"
    sed -i '' -e 's/[[:space:]]*\].*$//'  "$hostsfile"
    sed -i '' -e '/[[:space:]]*#.*$/ s/[[:space:]]*#.*$//'  "$hostsfile"
    sed -i '' -e '/^$/d' "$hostsfile"
    sed -i '' -e '/127.0.0.1/ s/127.0.0.1//'  "$hostsfile"
    sed -i '' -e '/192.168.1.2/ s/192.168.1.2//'  "$hostsfile"
    sed -i '' -e '/^www[0-9]./ s/^www[0-9].//'  "$hostsfile"
    sed -i '' -e '/^www./ s/^www.//' "$hostsfile"
    # удаляем дубликаты
    cat "$hostsfile" | sort -u > "$hostsfile".new
    mv "$hostsfile".new "$hostsfile"

    # including important informations
    echo  "##" >> "$endfile"
    echo  "##-----------------------------------------" >>"$endfile"
    echo  "##    Generated by hostblock.sh     " >>"$endfile"
    echo  "##    on $(date)" >>"$endfile"
    echo  "##    "`cat "$hostsfile" | wc -l`" blocked hosts" >>"$endfile"
    echo  "##-----------------------------------------" >>"$endfile"
    echo  "##" >>"$endfile"

    awk 'ORS=(NR%5)?"/":"/\n" ; BEGIN{end = 0} ; END{printf (NR%5)?"\n":""}' "$hostsfile" | sed "s_^_address=/_ ; s/$/$redirecturl/">>"$endfile"

    # rm -rf "$hostsfile"

    printf "\nMoving $endfile to /usr/local/etc/"
    cp "$endfile" /usr/local/etc/dnsmasq.hosts.conf

    # COMMANDS TO BE EXECUTED AFTER PROCESSING
    printf "\n\nRunning postprocessing..."
    postprocess && printf "done\n" || printf "FAILED"

    # CLEAN UP
    rm -r "$tmpdir"
else
    printf "\nDONE. No new changes."
fi
printf "\nHostsblock completed at `date +'%x %T'`\n"