creates ipset sets matching certain patterns in httpd logs

ipset-logpat.sh

#!/bin/bash

# ipset-logpat

# searches httpd access logs for pattern, whoises matching ip's and uses
# ip blocks to create ipset set. also adds iptables rules to log and reject
# requires: iptables, ipset, aggregate (optional)

# other useful ipset commands: ipset list [-terse], ipset destroy
# more info:
#   https://mattwilcox.net/web-development/unexpected-ddos-blocking-china-with-ipset-and-iptables
#   https://github.com/jordanrinke/ipsets-persistent
#   http://forums.debian.net/viewtopic.php?f=5&t=127437

# logs: you can use (...) syntax (array) for wildcards/globs like * or ? e.g.:
# logs=(/var/log/lighttpd/access.log*)
logs=(/var/log/lighttpd/access.log{,.1})

pattern="badhost.com"
whoisobj="^route:"
setname="badhost"
ipsout="/etc/iptables/${setname}.txt"
curdate="$( date +%y%m%d%H%M%S )"
stdlog="/var/log/ipset-${setname}.log"

func_help () {
cat <<-EOF
    
ipset-logpattern

Usage:

  "$0 [--cron|--quiet] [--iplog|--ipset|--iptables|--all]"

  [-c|--cron]	cron:		log stdout/err to "$stdlog"
  [-q|--quiet]	quiet:		dont output to stdout/err

  [-l|--iplog]	iplog:		create "$ipsout" using logs
  [-i|--ipset]	ipset:		ipset create "$setname"
  [-t|--ipset]	iptables:	insert iptables rule for ipset
  [-a|--all]	all:		all of the above (default w/o args)

Settings:

  variables can be edited in script or by using these arguments:
  [--logs|--pattern|--whoisobj|--setname|--ipsout|--stdlog] <value>

Example:

  "$0 -c --pattern "bad host" --setname badhost --ipsout /etc/badhosts.txt

EOF
}

std="/dev/stdin"
opt_ips="-quiet"

#  md5sum "${ipsout}"* | grep -v "${ipsout}$" | grep "$( md5sum "$ipsout" | awk '{ print $1 }' )" \
#   && echo "OK" || echo "NOK" 
func_ipl () {
  [ -s "$ipsout" ] && mv "$ipsout" "$ipsout.$curdate" || { [ -f $ipsout ] && rm "$ipsout"; }
  for i in $( zgrep "$pattern" "${logs[@]}" | cut -d":" -f2 | cut -d" " -f1 | sort -u ); do
    /sbin/ipset test "$opt_ips" "$setname" "$i" || { whois "$i" | grep "$whoisobj"; }
  done | awk '{ print $(NF) }' | sort -u >>"$ipsout"
  if /usr/bin/which aggregate >/dev/null 2>&1; then
    /usr/bin/aggregate -q < "$ipsout" >"${ipsout}.$$.tmp" && mv "${ipsout}.$$.tmp" "$ipsout"
  fi
  if [ -s "$ipsout" ]; then logc="OK - $( wc -l < "$ipsout" ) lines"
  else logc="NOK"; [ -f $ipsout ] && rm "$ipsout"; fi
  echo "$( date +%F\ %T ) iplog: create ip block list \"$ipsout\" - $logc"
}

func_ips () {
  /sbin/ipset create -exist "$opt_ips" "$setname" hash:net && ipsc="OK" || ipsc="NOK"
  echo "$( date +%F\ %T ) ipset: create set \"$setname\" - $ipsc"
  for i in ${ipsout}*; do
    while read l; do /sbin/ipset add -exist "$opt_ips" "$setname" "$l"; done < "$i" && \
      ipsa="OK" || ipsa="NOK"
  echo "$( date +%F\ %T ) ipset: add entries from \"$i\" to \"$setname\" - $ipsa"
  done 
}

func_ipt () {
  # with port: -A INPUT -p tcp -m tcp --dport 80 -m set --match-set $setname src -j LOGIPS
  # iptables rc: 0 chain already exists, 1 chain doesnt exist, grep match rc: 0 (exists)
  rulenum=3
  rules=(
    '-N LOGIPS'
    '-A LOGIPS -m limit --limit 10/min -j LOG --log-prefix "IPS REJECT: " --log-level 6'
    '-A LOGIPS -j REJECT --reject-with icmp-port-unreachable'
    '-I INPUT '"$rulenum"' -p tcp -m set --match-set '"$setname"' src -j LOGIPS'
  )
  rejrule="-I INPUT $rulenum -p tcp -m set --match-set $setname src -j REJECT --reject-with icmp-port-unreachable"
  i=0; while [ "$i" -lt "${#rules[@]}" ]; do
    act="$( echo ${rules[$i]} | cut -d" " -f1 )"; chain="$( echo ${rules[$i]} | cut -d" " -f2 )"
    if [ "$act" = "-N" ]; then
      /sbin/iptables -n -L "$chain" >/dev/null 2>&1; rc="$?"
    elif [ "$act" = "-A" ] || [ "$act" = "-I" ]; then
      target="$( echo "${rules[$i]}" | sed "s/-A\?I\? ${chain}.*\?-j \([A-Z]\+\)\( .*\|$\)/\1/" )"
      /sbin/iptables -n -L "$chain" | grep -q "^${target}"; rc="$?"
    fi
    if [ "$rc" -eq 0 ]; then res="already exists"; else eval /sbin/iptables "${rules[$i]}" && res="OK" || res="NOK"; fi
    if [ "$act" = "-N" ]; then action="add"; target="new"
      echo "$( date +%F\ %T ) iptables: create new $chain chain - $res"
    else
      if   [ "$act" = "-A" ]; then action="append"
      elif [ "$act" = "-I" ]; then action="insert"; fi
      echo "$( date +%F\ %T ) iptables: $action $target rule to $chain chain - $res"
    fi
    i=$((i+1))
  done
  /sbin/iptables -n -L "${rules[0]/#-? /}" >/dev/null 2>&1 || { \
    echo "$( date +%F\ %T ) iptables: could not create chain ${rules[0]/#-? /}, try replacement rule..."
    eval /sbin/iptables "$rejrule" && res="OK" || res="NOK"
    echo "$( date +%F\ %T ) iptables: insert REJECT rule to INPUT chain - $res"
  }
  unset act chain target rc res
}

# testing using $# 0 and (*) instead of the 'if' statement below
# if [ $# = 0 ]; { func_ipl; func_ips; func_ipt; } >>"$std" 2>&1; exit 0; fi
for (( ; $# >= 0; )); do
  case $1 in
    (start|restart|reload|force-reload) { func_ips; func_ipt; }; exit 0 ;;
    (flush) /sbin/ipset flush "$setname"; exit 0 ;;
    (stop)  echo "Automatic flushing disabled, use \"flush\" instead of \"stop\""; exit 0; ;;
    (-c|--cron)  std="$stdlog";   shift ;;
    (-q|--quiet) std="/dev/null"; shift ;;
    (-d|--debug) unset opt_ips;   shift ;;
    (--logs)     shift; logs="$1";     shift ;;
    (--whoisobj) shift; whoisobj="$1"; shift ;;
    (--pattern)  shift; pattern="$1";  shift ;;
    (--setname)  shift; setname="$1";  shift ;;
    (--ipsout)   shift; ipsout="$1";   shift ;;
    (--stdlog)   shift; stdlog="$1";   shift ;;
    (-a|--all)          { func_ipl; func_ips; func_ipt; } >>"$std" 2>&1; exit 0 ;;
    (-l|--iplog|--iplogs) func_ipl >>"$std" 2>&1; shift; exit 0 ;;
    (-i|--ips|--ipset)    func_ips >>"$std" 2>&1; shift; exit 0 ;;
    (-t|--ipt|--iptables) func_ipt >>"$std" 2>&1; shift; exit 0 ;;
    (--) shift; break ;;
    (-h|--help|-?|help)  func_help; exit 0 ;;
    (*) { func_ipl; func_ips; func_ipt; } >>"$std" 2>&1; exit 0 ;;
  esac
done