[LogstreamerInput]
log_directory="/var/log/heka"
file_match = "test-output.log"
journal_directory = "/var/spool/heka"
decoder = "RsyslogDecoder"
[RsyslogDecoder]
type = "SandboxDecoder"
filename = "lua_decoders/openstack.lua"
[RsyslogDecoder.config]
type = "RSYSLOG_ForwardFormat"
template = "<%PRI%>%TIMESTAMP:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg%"
[ESLogstashV0Encoder]
es_index_from_timestamp = true
type_name = "%{Type}"
[ESJsonEncoder]
index = "heka-%{2006.01.02}"
es_index_from_timestamp = true
type_name = "message"
[RstEncoder]
#[LogOutput]
#message_matcher = "TRUE"
#encoder = "RstEncoder"
[ElasticSearchOutput]
message_matcher = "TRUE"
server = "http://172.29.236.85:9200"
flush_interval = 5000
flush_count = 10
encoder = "ESLogstashV0Encoder"
[DashboardOutput]
ticker_interval = 5
[LevelCounter]
type = "SandboxFilter"
filename = "lua_filters/frequent_items.lua"
ticker_interval = 60
preserve_data = true
message_matcher = "Fields[os_level] != ''"
[LevelCounter.config]
message_variable = "Fields[os_level]"
max_items = 100
min_output_weight = 100
reset_days = 1-- This Source Code Form is subject to the terms of the Mozilla Public
-- License, v. 2.0. If a copy of the MPL was not distributed with this
-- file, You can obtain one at http://mozilla.org/MPL/2.0/.
--[[
Parses the rsyslog output using the string based configuration template.
Config:
- template (string)
The 'template' configuration string from rsyslog.conf.
http://rsyslog-5-8-6-doc.neocities.org/rsyslog_conf_templates.html
- tz (string, optional, defaults to UTC)
If your rsyslog timestamp field in the template does not carry zone offset information, you may set an offset
to be applied to your events here. Typically this would be used with the "Traditional" rsyslog formats.
Parsing is done by `Go <http://golang.org/pkg/time/#LoadLocation>`_, supports values of "UTC", "Local",
or a location name corresponding to a file in the IANA Time Zone database, e.g. "America/New_York".
*Example Heka Configuration*
.. code-block:: ini
[RsyslogDecoder]
type = "SandboxDecoder"
filename = "lua_decoders/rsyslog.lua"
[RsyslogDecoder.config]
type = "RSYSLOG_TraditionalFileFormat"
template = '%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n'
tz = "America/Los_Angeles"
*Example Heka Message*
:Timestamp: 2014-02-10 12:58:58 -0800 PST
:Type: RSYSLOG_TraditionalFileFormat
:Hostname: trink-x230
:Pid: 0
:UUID: e0eef205-0b64-41e8-a307-5772b05e16c1
:Logger: RsyslogInput
:Payload: "imklog 5.8.6, log source = /proc/kmsg started."
:EnvVersion:
:Severity: 7
:Fields:
| name:"programname" value_string:"kernel"
--]]
local syslog = require "syslog"
local l = require 'lpeg'
local dt = require 'date_time'
local table = require 'table'
local template = read_config("template")
local msg_type = read_config("type")
local msg = {
Timestamp = nil,
Type = msg_type,
Hostname = nil,
Payload = nil,
Pid = nil,
Severity = nil,
Fields = nil
}
function string:split( inSplitPattern, outResults )
if not outResults then
outResults = { }
end
local theStart = 1
local theSplitStart, theSplitEnd = string.find( self, inSplitPattern, theStart, true )
while theSplitStart do
table.insert( outResults, string.sub( self, theStart, theSplitStart-1 ) )
theStart = theSplitEnd + 1
theSplitStart, theSplitEnd = string.find( self, inSplitPattern, theStart, true )
end
table.insert( outResults, string.sub( self, theStart ) )
return outResults
end
local sp = l.space
local greedy = l.R("az", "AZ", "09", "..", "__", "--")^1
local levels = l.Cg(
l.P"TRACE"
+ l.P"ERROR"
+ l.P"DEBUG"
+ l.P"INFO"
+ l.P"AUDIT"
+ l.P"CRITICAL"
+ l.P"WARNING"
, "OpenStack_Level")
local pid = l.Cg(l.R("09")^1, "OpenStack_PID")
local os_function = l.Cg(greedy, "OpenStack_Function")
local os_ts = l.Ct(dt.rfc3339_full_date * sp * dt.rfc3339_partial_time)
local os_ns = l.Cg(os_ts / dt.time_to_ns, "OpenStack_TimeStamp")
local os_msg = l.Cg(l.P(1)^0, "OpenStack_Message")
local os_fields = l.Ct(os_ns * sp * pid * sp * levels * sp * os_function * sp * os_msg)
local grammar = syslog.build_rsyslog_grammar(template)
-- grammar = l.Ct(os_fields)
function process_message ()
local log = read_message("Payload")
local fields = grammar:match(log)
if not fields then return -1 end
local m = os_fields:match(fields.msg)
if m then
fields.os_level = m.OpenStack_Level
fields.os_pid = m.OpenStack_PID
fields.os_function = m.OpenStack_Function
fields.os_ts = m.OpenStack_TimeStamp
fields.os_msg = m.OpenStack_Message
end
if fields.timestamp then
msg.Timestamp = fields.timestamp
fields.timestamp = nil
end
if fields.pri then
msg.Severity = fields.pri.severity
fields.syslogfacility = fields.pri.facility
msg.Pri = fields.pri
fields.pri = nil
else
msg.Severity = fields.syslogseverity or fields["syslogseverity-text"]
or fields.syslogpriority or fields["syslogpriority-text"]
fields.syslogseverity = nil
fields["syslogseverity-text"] = nil
fields.syslogpriority = nil
fields["syslogpriority-text"] = nil
end
if fields.syslogtag then
fields.programname = fields.syslogtag.programname
msg.Pid = fields.syslogtag.pid
fields.syslogtag = nil
end
if fields.programname then
local container_strip = fields.programname:split('.')
fields.os_container = container_strip[1]
fields.os_service = container_strip[2]
end
msg.Hostname = fields.hostname or fields.source
fields.hostname = nil
fields.source = nil
msg.Payload = fields.msg
fields.msg = nil
msg.Fields = fields
inject_message(msg)
return 0
end