dd-wrt adblock script

gen_host.sh

#!/bin/sh

##################################################################################
##
## gen_hosts by IronManLok
##
##   Downloads domain entries of known ad abusers from multiple sources, 
##   cleans up, merges and removes duplicates. Includes white-listing and
##   custom host entries.
##   
##   This script is intended to be used on units running DD-WRT, it requires
##   the use of JFFS (or USB drive mounted on /jffs) and DNSMasq as DNS server.
##
##   On Services Tab, at Additional DNSMasq options, add this line:
##      addn-hosts=/tmp/gen_host.txt
##
##   Call this script from your firewall script. Also, use cron to schedule its
##   execution. For running everyday at 22:00:
##      0 22 * * * root /jffs/gen_host.sh
##
##   For white-listing, create /jffs/whitelist_hosts.txt and list one domain
##   per line. For custom hosts entries, create /jffs/my_hosts.txt and 
##   add any lines in the same format of a regular hosts file.
##
##   This script is free for use, modification and redistribution as long as
##   appropriate credit is provided.
##
##   THIS SCRIPT IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, BUT WITHOUT 
##   ANY WARRANTY. IT IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER 
##   EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
##   OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS
##   TO THE QUALITY AND PERFORMANCE OF THE SCRIPT IS WITH YOU. SHOULD THE SCRIPT
##   PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR 
##   CORRECTION.
##
##################################################################################

wait_for_connection() {
  while :; do
    ping -c 1 -w 10 www.google.com > /dev/null 2>&1 && break
    sleep 60
    logger "gen_host: Retrying internet connection..."
  done
}

# Makes sure only one instance of this script is running
if test -s /tmp/gen_host.lck; then
  logger "gen_host: Already running, quitting."
  exit 1
fi

echo $$ > /tmp/gen_host.lck

logger "gen_host: Generating hosts file..."

if test -s /tmp/gen_host.txt; then
  rm /tmp/gen_host.txt
fi

if test -s /tmp/gen_host.tmp; then
  rm /tmp/gen_host.tmp
fi

wait_for_connection

COUNT=1

# The script must run within 900 seconds, this will create a timer to terminate it
(sleep 900 && logger "gen_host: Execution timed out." && rm /tmp/gen_host.lck && kill -TERM $$) & TIMEOUT_PID=$!

for URL in "http://winhelp2002.mvps.org/hosts.txt" \
           "http://someonewhocares.org/hosts/zero/hosts" \
           "http://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&mimetype=plaintext" \
           "http://jansal.googlecode.com/svn/trunk/adblock/hosts" \
           "http://hosts-file.net/ad_servers.txt" \
           "http://adblock.gjtech.net/?format=hostfile" \
           "http://www.hostsfile.org/Downloads/hosts.txt"; do
  ATTEMPT=1
  while :; do
    logger "gen_host: Downloading host entries from $URL (attempt `echo $ATTEMPT`)..."

    # Each file should be downloaded within 120 seconds
    TEMP_FILE="/tmp/gen_host`echo $COUNT`.tmp"
    (wget -q -O - "$URL" > "$TEMP_FILE") & DOWNLOAD_PID=$!
    (sleep 120 && kill -TERM $DOWNLOAD_PID) & WAITER_PID=$!
    wait $DOWNLOAD_PID
    RESULT=$?
    kill -KILL $WAITER_PID

    # Clean-up:
    #  1) removes CR
    #  2) converts double spaces/tabs to single tab
    #  3) removes leading spaces
    #  4) removes trailing spaces
    #  5) removes empty lines
    #  6) removes fully commented lines
    #  7) removes trailing comments
    #  8) removes invalid characters
    #  9) replaces 127.0.0.1 with 0.0.0.0
    # 10) removes non-leading 127.0.0.1 or 0.0.0.0
    # 11) keeps only valid 0.0.0.0 entries
    # 12) removes any lines with localhost
    # 13) breaks up multiple entries on a single line into several single entry lines

    if [ $RESULT = 0 ]; then
      cat "$TEMP_FILE" | tr -d '\015' | \
                         sed -r -e 's/[[:space:]]+/\t/g' \
                                -e 's/^\t//g' \
                                -e 's/\t$//g' \
                                -e '/^$/d' \
                                -e '/^#/d' \
                                -e 's/\t*#.*$//g' \
                                -e 's/[^a-zA-Z0-9\.\_\t\-]//g' \
                                -e 's/^127\.0\.0\.1/0.0.0.0/g' \
                                -e 's/\t(0\.0\.0\.0|127\.0\.0\.1)//g' | \
                         grep ^0'\.'0'\.'0'\.'0$'\t'. | \
                         grep -v -F localhost | \
                         sed -e 's/^0\.0\.0\.0\t/0.0.0.0%/1' -e 's/\t/%%0\.0\.0\.0\t/g' -e 's/^0\.0\.0\.0%/0.0.0.0\t/1' -e 's/%%/\n/g' \
                         >> /tmp/gen_host.tmp
      rm "$TEMP_FILE"
      break
    fi

    ATTEMPT=$(($ATTEMPT + 1))
    
    # Should attempt to download a file 3 times. If it fails on all 3, try again after 20 minutes...
    if [ $ATTEMPT = 4 ]; then
      logger "gen_host: Download failed, retrying in 20 minutes..."
      (sleep 1200 && /jffs/gen_host.sh) &
      rm /tmp/gen_host.lck
      kill -KILL $TIMEOUT_PID
      exit 2
    fi

    sleep 10
  done

  COUNT=$(($COUNT + 1))

done

# Add custom host entries to the file
if test -s /jffs/my_hosts.txt; then
  logger "gen_host: Adding custom host entries..."
  cat /jffs/my_hosts.txt >> /tmp/gen_host.tmp
fi

# Remove white-listed entries
if test -s /jffs/whitelist_hosts.txt; then
  logger "gen_host: Removing white-listed entries..."

  ORIGIN_FILE="/tmp/gen_host.tmp"

  for WHITELIST in `cat /jffs/whitelist_hosts.txt`; do
    COUNT=$(($COUNT + 1))
    TEMP_FILE="/tmp/gen_host`echo $COUNT`.tmp"
    grep -v "^0\.0\.0\.0\t$WHITELIST\$" "$ORIGIN_FILE" > "$TEMP_FILE"
    rm "$ORIGIN_FILE"
    ORIGIN_FILE="$TEMP_FILE"
  done

  if [ "$ORIGIN_FILE" != "/tmp/gen_host.tmp" ]; then
    mv "$ORIGIN_FILE" /tmp/gen_host.tmp
  fi
fi

# Removing duplicates, use awk in case your build of DD-WRT doesn't have sort
logger "gen_host: Removing duplicate entries..."
## awk '!x[$0]++' /tmp/gen_host.tmp > /tmp/gen_host.txt
sort -u /tmp/gen_host.tmp > /tmp/gen_host.txt
rm /tmp/gen_host.tmp

logger "gen_host: Generated `wc -l < /tmp/gen_host.txt` domain entries. Restarting DNSMasq..."

stopservice dnsmasq
startservice dnsmasq

rm /tmp/gen_host.lck
kill -KILL $TIMEOUT_PID