heka TOML samples

heka.sample.toml

[vagrant@hekad heka]$ cat global.toml
[hekad]
maxprocs = 1
pid_file = "/var/run/hekad.pid"
# heka needs rw
base_dir = "/var/cache/hekad"
# heka needs ro
share_dir = "/usr/share/heka"

[DashboardOutput]
ticker_interval = 10



[vagrant@hekad heka]$ cat system-syslog.toml
[syslog_logs]
type = "LogstreamerInput"
parser_type = "token"
decoder = "rsyslog_decoder"
log_directory = "/var/log/"
# Needs to be single-quoted
# See LogstreamerInput documantation
file_match = 'messages\.?(?P<seq>\d*)'
priority = ["^seq"]

[rsyslog_decoder]
type = "SandboxDecoder"
filename = "lua_decoders/rsyslog.lua"

[rsyslog_decoder.config]
type = "syslog"
template = '%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n'


[auth_logs]
type = "LogstreamerInput"
parser_type = "token"
decoder = "rsyslog_decoder_auth"
log_directory = "/var/log/"
# Needs to be single-quoted
# See LogstreamerInput documantation
file_match = 'secure\.?(?P<seq>\d*)'
priority = ["^seq"]

[rsyslog_decoder_auth]
type = "SandboxDecoder"
filename = "lua_decoders/rsyslog.lua"

[rsyslog_decoder_auth.config]
type = "auth"
template = '%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n'


[es_encoder]
type = "ESJsonEncoder"
index = "system-%{Type}_"
es_index_from_timestamp = false
type_name = "heka-%{Type}"

[es_output]
type = "ElasticSearchOutput"
message_matcher = "Type == 'syslog' || Type == 'auth'"
encoder = "es_encoder"
flush_interval = 10000
flush_count = 50
server = "http://es.io-test.ripe.net:9200"



[vagrant@hekad heka]$ cat hdp-json.toml
[hdp_tag]
type = "ScribbleDecoder"
[hdp_tag.message_fields]
tag = "hdfs-dn"

[hdp_log]
type = "LogstreamerInput"
parser_type = "token"
log_directory = "/var/log/"
# Needs to be single-quoted
# See LogstreamerInput documantation
file_match = '.*.json\.?(?P<seq>\d*)'
priority = ["^seq"]
decoder = "hdp_tag"

[es_hdp_payload]
type = "SandboxEncoder"
filename = "lua_encoders/es_payload.lua"
    [es_hdp_payload.config]
    #type = "yarn_logs"
    es_index_from_timestamp = false
    index = "cluster_"
    type_name = "hdp-log"

[es_hdp_output]
type = "ElasticSearchOutput"
message_matcher = "Logger =~ /hdp_/"
encoder = "es_hdp_payload"
flush_interval = 10000
flush_count = 50
server = "http://es.io-test.ripe.net:9200"