hadoop-logaggr-timeline.sh

#!/bin/bash

aggregate_min=30 # default value

usage()
{
    echo "hadoop-logaggr-timeline.sh [-h] [-t N] file" >&2
    echo "  -t N[min]: must be integer (default:30) " >&2
    echo "            aggregates logs each N minutes." >&2
    echo "         -h: help (this message)" >&2
    exit 0
}

TEMP=`getopt t:h $*`

if [ $? != 0 ] ; then
    usage
fi

eval set -- "$TEMP"

while true ; do
        case "$1" in
                -t)
		aggregate_min=$2
		shift 2;;
                -h|--help)
		usage
		shift ;;
                --) shift ; break ;;
                *)  break ;;
        esac
done

if [ $# != 1 ] ; then
    usage
fi

# grep: remove all lines which doesn't have datetime like YYYY-MM-DD hh:mm:ss
# sed: delete other part of datetime like YYYY-MM-DD hh:mm:ss
# python: aggregating and visualizing
cat $1  \
| grep "^[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\} [0-9]\{2\}:[0-9]\{2\}" \
| sed "s/\(^[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\} [0-9]\{2\}:[0-9]\{2\}\).*/\1/" \
| uniq -c | python -c "
import sys
import math

if sys.argv[1].isdigit():
    AGGREGATE_MIN=int(sys.argv[1])
else:
    AGGREGATE_MIN=30
   

max_count = 0
timedict = {}
for line in sys.stdin:
    arr = line.strip().split()
    count = int(arr.pop(0))
    date = arr.pop(0)
    time = arr.pop(0)
    time_list = time.split(':')
    time_list[1] = str(int(time_list[1])/AGGREGATE_MIN*AGGREGATE_MIN)
    time = ':'.join(time_list)
    datetime = '%s %s' % (date, time)
    timedict[datetime] = timedict.get(datetime,0) + count
    if timedict[datetime] > max_count:
        max_count = timedict[datetime]
    

max_count_char_num = int(math.ceil(math.log(max_count,10)))

for time,count in timedict.iteritems():
    count_normalized = int(float(count)/max_count * 30)
    padding = ' ' * (max_count_char_num - len(str(count)))
    print '%-16s : %s%d %s' % (time, padding, count, '*'*count_normalized)
" ${aggregate_min} | sort | less