nginx access log collector for graphite; parses a slightly modified "combined" format (`$request_time` added at the end) and pushes 30 second counts and request time averages via UDP, using the line protocol; needs an awk that has mktime (like gawk)

accesslog_collector.sh

#!/bin/bash
#
#   Copyright 2013 Sabin Iacob <[email protected]>
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

app=$1
server=${2:-127.0.0.1}
input=${3:-/home/$app/logs/access.log}

LOG_PROCESSOR='
function month(txt) {
    months["Jan"] = 1;
    months["Feb"] = 2;
    months["Mar"] = 3;
    months["Apr"] = 4;
    months["May"] = 5;
    months["Jun"] = 6;
    months["Jul"] = 7;
    months["Aug"] = 8;
    months["Sep"] = 9;
    months["Oct"] = 10;
    months["Nov"] = 11;
    months["Dec"] = 12;

    return months[txt];
}

function ts(dt){
    oldfs = FS
    FS = "[ :/\[]";
    $0 = dt;
    y = $4;
    m = month($3);
    d = $2;
    h = $5;
    min = $6;
    s = $7;

    FS=oldfs

    sub("^0", "", d);
    sub("^0", "", h);
    sub("^0", "", min);
    sub("^0", "", s);
    
    return mktime(y " " m " " d " " h " " min " " s);
}

/[0-9.]$/ {
    rtime = $NF;
    if(rtime > 0) {
        cnt += 1;
        sum += rtime;
        tstamp = ts($4);

        if(tstamp - prev_ts >= 30 && cnt > 0) {
            avg = sum / cnt;
            print "'$app'.nginx.rtime", avg, tstamp;
            print "'$app'.nginx.requests", cnt / 30, tstamp;
            fflush();
            cnt = 0;
            sum = 0;
            prev_ts = tstamp;
        }
    }
}
'

set -e 
trap 'kill 0' EXIT

if [ "$input" = "-" ]; then
    /usr/bin/awk "$LOG_PROCESSOR" >/dev/tcp/$server/2003 # importing logs; we actually care if they reach the database
else
    (/usr/bin/tail -F $input | /usr/bin/awk "$LOG_PROCESSOR") >/dev/udp/$server/2003 # streaming logs; we don't care if some packets don't make it through
fi