Heka haproxy lua decoder

haproxy.lua

local dt = require "date_time"
local ip = require "ip_address"
local l = require 'lpeg'
local syslog = require "syslog"
l.locale(l)

local msg = {
Timestamp   = nil,
Hostname    = nil,
Payload     = nil,
Pid         = nil,
Fields      = nil
}

local syslog_grammar = syslog.build_rsyslog_grammar("%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n")
local sp = l.space
local timestamp = "[" * l.Cg(dt.build_strftime_grammar("%d/%b/%Y:%H:%M:%S") * dt.time_secfrac / dt.time_to_ns, "Timestamp") * "]"
local log_date = l.Cg(dt.rfc3164_timestamp, "LogDate")
local host = (l.alnum^1 + l.S("-_"))^1

local proc = l.P"haproxy[" * l.R("09")^1 * l.P"]:" * l.Cg(l.Cc"haproxy", "Type")
local remote_addr = l.Cg(ip.v4, "remote_addr") * ":" * l.Cg(l.R("09")^1, "port")
-- Ommit this for the default haproxy log format
local uuid = "[" * l.Cg((l.R("AZ", "09") + l.S":_")^1, "uuid") * "]"
local request = l.P{'"' * l.Cg(((1 - l.P'"') + l.V(1))^0, "request") * '"'}
local status = l.Cg(l.digit * l.digit * l.digit, "status")
local bytes = l.Cg(l.digit^1, "bytes")
local srv_data = host * sp * host * l.P"/" * host
local ha_shit = (l.digit^1 + l.P"/")^1
local sep = "- - ----"

local msg_grammar = l.Ct(log_date * sp * host * sp * proc * sp * remote_addr * sp * timestamp * sp * uuid * sp * srv_data * sp * ha_shit * sp * status * sp * bytes * sp * sep * sp * ha_shit * sp * ha_shit * sp * request)

function process_message ()
    local log = read_message("Payload")
    local fields = syslog_grammar:match(log)
    if not fields then return -1 end

    msg.Timestamp = fields.timestamp
    fields.timestamp = nil

    fields.programname = fields.syslogtag.programname
    msg.Pid = fields.syslogtag.pid or nil
    fields.syslogtag = nil

    msg.Hostname = fields.hostname
    fields.hostname = nil

    local m = msg_grammar:match(fields.msg)
    if m then
        msg.Type = m.Type
        msg.Payload = nil
        msg.Timestamp = m.Timestamp
        fields.uuid = m.uuid
        fields.remote_addr = m.remote_addr
        fields.request = m.request
        fields.status = m.status
        fields.bytes = m.bytes
    else
       -- Fail with return -1 or do whatever you want
       msg.Type = "Ignore"
       msg.Payload = fields.msg
    end
    fields.msg = nil

    msg.Fields = fields
    inject_message(msg)
    return 0
end

Test it http://lpeg.trink.com/share/13911645150047197117

Mine is here: https://gist.github.com/asyd/bf0ce751c3000b2214ac i added the HTTP Host: header, and fix it for 0.10.0b

@asyd that's great!

@asyd I've fixed a bug in the date parsing, strftime doesn't take into account rfc3164 buggy day logging, but @trink took this into account when building the datetime lua module: https://github.com/mozilla-services/lua_sandbox/blob/master/modules/date_time.lua#L250