Last active
February 16, 2016 07:58
-
-
Save ghostflare76/2a636d26dcfa7598cf1e to your computer and use it in GitHub Desktop.
fluentd config
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# nginx log -> elasticsearch | |
# nginx log -> rabbitmq | |
# nginx log -> norikra -> email | |
# nginx log -> norikra -> elasticsearch | |
# spring metrcis log -> graphite | |
# metrics log ago | |
2016-02-16 06:55:15 +0900 event.log: {"date":"2016-02-16","times":"06:55:15,344","level":"INFO","type":"COUNTER","name":"gc.ps_marksweep.count","count":"2"} | |
2016-02-16 06:55:15 +0900 event.log: {"date":"2016-02-16","times":"06:55:15,344","level":"INFO","type":"COUNTER","name":"gc.ps_marksweep.time","count":"213"} | |
2016-02-16 06:55:15 +0900 event.log: {"date":"2016-02-16","times":"06:55:15,344","level":"INFO","type":"COUNTER","name":"gc.ps_scavenge.count","count":"114"} | |
2016-02-16 06:55:15 +0900 event.log: {"date":"2016-02-16","times":"06:55:15,344","level":"INFO","type":"COUNTER","name":"gc.ps_scavenge.time","count":"672"} | |
# metrics log after | |
2016-02-16 16:28:15 +0900 graphite.opdev-infra01: {"gc.ps_marksweep.count":2} | |
2016-02-16 16:28:15 +0900 graphite.opdev-infra01: {"gc.ps_marksweep.time":213} | |
2016-02-16 16:28:15 +0900 graphite.opdev-infra01: {"gc.ps_scavenge.count":114} | |
2016-02-16 16:28:15 +0900 graphite.opdev-infra01: {"gc.ps_scavenge.time":672} | |
<source> | |
type tail | |
path /var/log/foo/bar.log | |
pos_file /var/log/td-agent/foo-bar.log.pos | |
tag foo.bar | |
format /(?<date>[^ ]*) (?<times>[^ ]*) (?<level>[^ ]*) (([^=]+))=(?<type>([^=]+))(?:,|$) (([^=]+))=(?<name>([^=]+))(?:,|$) (([^=]+))=(?<count>([^=]+))(?:,|$)/ | |
</source> | |
<source> | |
type tail | |
path /home/opdev/logs/nginx/opdev.pinpoint.korea.ncsoft.corp_access.log | |
pos_file /var/log/td-agent/opdev.pinpoint.access.log.pos | |
format /^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/ | |
time_format %d/%b/%Y:%H:%M:%S %z | |
tag nginx.access | |
</source> | |
<filter nginx.access> | |
type grep | |
exclude1 path css|js|png|gif|jpg|jpeg | |
</filter> | |
<match nginx.access> | |
type copy | |
<store> | |
type stdout | |
</store> | |
<store> | |
type norikra | |
norikra 127.0.0.1:26571 | |
buffer_queue_limit 64 | |
retry_limit 17 | |
remove_tag_prefix nginx | |
target_map_tag true | |
</store> | |
<store> | |
type elasticsearch | |
host 127.0.0.1 | |
port 9200 | |
logstash_format true | |
type_name pinpoint | |
logstash_prefix nginx_access | |
</store> | |
<store> | |
#rabbitmq 내에서 routing key는 tag 네임으로 정의해서 bind 해서 사용해야 수집가능 | |
type amqp | |
host 127.0.0.1 | |
port 5672 | |
user guest | |
password guest | |
exchange fluentd_exchange | |
exchange_type topic | |
exchange_durable true | |
payload_only false | |
content_type text/x-json #application/json | |
</store> | |
</match> | |
<source> | |
type norikra | |
norikra 127.0.0.1:26571 | |
<fetch> | |
method sweep | |
tag query_name | |
tag_prefix norikra.query | |
interval 10s | |
</fetch> | |
</source> | |
<match norikra.query.access**> | |
type copy | |
<store> | |
type elasticsearch | |
host 172.20.54.152 | |
port 9200 | |
logstash_format true | |
type_name pinpoint | |
logstash_prefix norikra_access | |
</store> | |
<store> | |
type mail | |
host mail.host.com | |
port 25 | |
from [email protected] | |
to [email protected] | |
subject SUBJECT: %s | |
subject_out_keys tag | |
out_keys tag,requests,remote | |
</store> | |
</match> | |
<source> | |
type tail | |
path /home/opdev/GPORTAL/logs/metrics.out | |
pos_file /var/log/td-agent/metrics.out.pos | |
format /(?<logtime>[^ ]* [^ ]*) (?<level>[^ ]*) (([^=]+))=(?<type>([^=]+))(?:,|$) (([^=]+))=(?<name>([^=]+))(?:,|$) (([^=]+))=(?<count>([^=]+))(?:,|$)/ | |
time_format %Y-%m-%d %H:%M:%S | |
tag event.log | |
</source> | |
<match event.log> | |
type copy | |
<store> | |
type map | |
map ([["graphite.#{Socket.gethostname}", time, {record["name"] => record["count"].to_i}]]) | |
multi true | |
</store> | |
</match> | |
<match graphite **> | |
type copy | |
<store> | |
type stdout | |
</store> | |
<store> | |
type graphite | |
host 127.0.0.1 | |
port 2003 | |
name_keys gc.ps_marksweep.count,gc.ps_marksweep.time,gc.ps_scavenge.count,gc.ps_scavenge.time,heap.committed,heap.init,heap.used,threads.daemon,threads.peak,threads.totalStarted | |
</store> | |
</match> |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment