Skip to content

Instantly share code, notes, and snippets.

@d34dh0r53
Last active August 29, 2015 14:17
Show Gist options
  • Save d34dh0r53/3641add904e2d046b576 to your computer and use it in GitHub Desktop.
Save d34dh0r53/3641add904e2d046b576 to your computer and use it in GitHub Desktop.
Heka Notes

Heka Notes

Configuration Files

[LogstreamerInput]
log_directory="/var/log/heka"
file_match = "test-output.log"
journal_directory = "/var/spool/heka"
decoder = "RsyslogDecoder"

[RsyslogDecoder]
type = "SandboxDecoder"
filename = "lua_decoders/openstack.lua"

[RsyslogDecoder.config]
type = "RSYSLOG_ForwardFormat"
template = "<%PRI%>%TIMESTAMP:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg%"

[ESLogstashV0Encoder]
es_index_from_timestamp = true
type_name = "%{Type}"

[ESJsonEncoder]
index = "heka-%{2006.01.02}"
es_index_from_timestamp = true
type_name = "message"

[RstEncoder]

#[LogOutput]
#message_matcher = "TRUE"
#encoder = "RstEncoder"

[ElasticSearchOutput]
message_matcher = "TRUE"
server = "http://172.29.236.85:9200"
flush_interval = 5000
flush_count = 10
encoder = "ESLogstashV0Encoder"

[DashboardOutput]
ticker_interval = 5

[LevelCounter]
type = "SandboxFilter"
filename = "lua_filters/frequent_items.lua"
ticker_interval = 60
preserve_data = true
message_matcher = "Fields[os_level] != ''"

[LevelCounter.config]
message_variable = "Fields[os_level]"
max_items = 100
min_output_weight = 100
reset_days = 1

OpenStack Filters

-- This Source Code Form is subject to the terms of the Mozilla Public
-- License, v. 2.0. If a copy of the MPL was not distributed with this
-- file, You can obtain one at http://mozilla.org/MPL/2.0/.

--[[
Parses the rsyslog output using the string based configuration template.

Config:

- template (string)
    The 'template' configuration string from rsyslog.conf.
    http://rsyslog-5-8-6-doc.neocities.org/rsyslog_conf_templates.html

- tz (string, optional, defaults to UTC)
    If your rsyslog timestamp field in the template does not carry zone offset information, you may set an offset
    to be applied to your events here. Typically this would be used with the "Traditional" rsyslog formats.

    Parsing is done by `Go <http://golang.org/pkg/time/#LoadLocation>`_, supports values of "UTC", "Local",
    or a location name corresponding to a file in the IANA Time Zone database, e.g. "America/New_York".

*Example Heka Configuration*

.. code-block:: ini

    [RsyslogDecoder]
    type = "SandboxDecoder"
    filename = "lua_decoders/rsyslog.lua"

    [RsyslogDecoder.config]
    type = "RSYSLOG_TraditionalFileFormat"
    template = '%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n'
    tz = "America/Los_Angeles"

*Example Heka Message*

:Timestamp: 2014-02-10 12:58:58 -0800 PST
:Type: RSYSLOG_TraditionalFileFormat
:Hostname: trink-x230
:Pid: 0
:UUID: e0eef205-0b64-41e8-a307-5772b05e16c1
:Logger: RsyslogInput
:Payload: "imklog 5.8.6, log source = /proc/kmsg started."
:EnvVersion:
:Severity: 7
:Fields:
    | name:"programname" value_string:"kernel"
--]]

local syslog = require "syslog"
local l = require 'lpeg'
local dt = require 'date_time'
local table = require 'table'

local template = read_config("template")
local msg_type = read_config("type")

local msg = {
Timestamp   = nil,
Type        = msg_type,
Hostname    = nil,
Payload     = nil,
Pid         = nil,
Severity    = nil,
Fields      = nil
}

function string:split( inSplitPattern, outResults )
  if not outResults then
    outResults = { }
  end
  local theStart = 1
  local theSplitStart, theSplitEnd = string.find( self, inSplitPattern, theStart, true )
  while theSplitStart do
    table.insert( outResults, string.sub( self, theStart, theSplitStart-1 ) )
    theStart = theSplitEnd + 1
    theSplitStart, theSplitEnd = string.find( self, inSplitPattern, theStart, true )
  end
  table.insert( outResults, string.sub( self, theStart ) )
  return outResults
end

local sp = l.space
local greedy = l.R("az", "AZ", "09", "..", "__", "--")^1
local levels = l.Cg(
    l.P"TRACE"
  + l.P"ERROR"
  + l.P"DEBUG"
  + l.P"INFO"
  + l.P"AUDIT"
  + l.P"CRITICAL"
  + l.P"WARNING"
  , "OpenStack_Level")
local pid = l.Cg(l.R("09")^1, "OpenStack_PID")
local os_function = l.Cg(greedy, "OpenStack_Function")
local os_ts = l.Ct(dt.rfc3339_full_date * sp * dt.rfc3339_partial_time)
local os_ns = l.Cg(os_ts / dt.time_to_ns, "OpenStack_TimeStamp")
local os_msg = l.Cg(l.P(1)^0, "OpenStack_Message")

local os_fields = l.Ct(os_ns * sp * pid * sp * levels * sp * os_function * sp * os_msg)


local grammar = syslog.build_rsyslog_grammar(template)

-- grammar = l.Ct(os_fields)

function process_message ()
    local log = read_message("Payload")
    local fields = grammar:match(log)
    if not fields then return -1 end

    local m = os_fields:match(fields.msg)
    if m then
        fields.os_level = m.OpenStack_Level
        fields.os_pid = m.OpenStack_PID
        fields.os_function = m.OpenStack_Function
        fields.os_ts = m.OpenStack_TimeStamp
        fields.os_msg = m.OpenStack_Message
    end

    if fields.timestamp then
        msg.Timestamp = fields.timestamp
        fields.timestamp = nil
    end

    if fields.pri then
        msg.Severity = fields.pri.severity
        fields.syslogfacility = fields.pri.facility
        msg.Pri = fields.pri
        fields.pri = nil
    else
        msg.Severity = fields.syslogseverity or fields["syslogseverity-text"]
        or fields.syslogpriority or fields["syslogpriority-text"]

        fields.syslogseverity = nil
        fields["syslogseverity-text"] = nil
        fields.syslogpriority = nil
        fields["syslogpriority-text"] = nil
    end

    if fields.syslogtag then
        fields.programname = fields.syslogtag.programname
        msg.Pid = fields.syslogtag.pid
        fields.syslogtag = nil
    end

    if fields.programname then
        local container_strip = fields.programname:split('.')
        fields.os_container = container_strip[1]
        fields.os_service = container_strip[2]
    end

    msg.Hostname = fields.hostname or fields.source
    fields.hostname = nil
    fields.source = nil

    msg.Payload = fields.msg
    fields.msg = nil
    msg.Fields = fields
    inject_message(msg)
    return 0
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment