visitor-counter.sh

#!/bin/bash
#
# Tech and Me 2015 - https://www.en0ch.se

visitor="/var/log/nginx/access.log"

lynx -source http://www.reportsfromearth.com/1140/names-best-known-bots-spiders-crawlers-visiting-website-2014/|grep "<td>"|grep -vE '("<td>"|><)'|sed 's/<td>//g'|sed 's|</td>||g'|sort|uniq|awk 'length >= 4'|grep -ivE '(:|gif|wget|xget)'>.rr_tmp
bnames=$(wc -l .rr_tmp|awk '{print $1}');echo "found ${bnames} known crawlers"
grep -ivf .rr_tmp ${visitor}>.rr1_tmp
rnames=$(wc -l .rr1_tmp|awk '{print $1}');echo "found ${rnames} matching log entries"
unames=$(awk '{print $1}' .rr1_tmp|sort|uniq|wc -l|awk '{print $1}')
echo "total ${unames} unique hits"

rm -f .rr_tmp .rr1_tmp

exit 0