Skip to content

Instantly share code, notes, and snippets.

@ghostflare76
Last active February 16, 2016 07:58
Show Gist options
  • Save ghostflare76/2a636d26dcfa7598cf1e to your computer and use it in GitHub Desktop.
Save ghostflare76/2a636d26dcfa7598cf1e to your computer and use it in GitHub Desktop.
fluentd config
# nginx log -> elasticsearch
# nginx log -> rabbitmq
# nginx log -> norikra -> email
# nginx log -> norikra -> elasticsearch
# spring metrcis log -> graphite
# metrics log ago
2016-02-16 06:55:15 +0900 event.log: {"date":"2016-02-16","times":"06:55:15,344","level":"INFO","type":"COUNTER","name":"gc.ps_marksweep.count","count":"2"}
2016-02-16 06:55:15 +0900 event.log: {"date":"2016-02-16","times":"06:55:15,344","level":"INFO","type":"COUNTER","name":"gc.ps_marksweep.time","count":"213"}
2016-02-16 06:55:15 +0900 event.log: {"date":"2016-02-16","times":"06:55:15,344","level":"INFO","type":"COUNTER","name":"gc.ps_scavenge.count","count":"114"}
2016-02-16 06:55:15 +0900 event.log: {"date":"2016-02-16","times":"06:55:15,344","level":"INFO","type":"COUNTER","name":"gc.ps_scavenge.time","count":"672"}
# metrics log after
2016-02-16 16:28:15 +0900 graphite.opdev-infra01: {"gc.ps_marksweep.count":2}
2016-02-16 16:28:15 +0900 graphite.opdev-infra01: {"gc.ps_marksweep.time":213}
2016-02-16 16:28:15 +0900 graphite.opdev-infra01: {"gc.ps_scavenge.count":114}
2016-02-16 16:28:15 +0900 graphite.opdev-infra01: {"gc.ps_scavenge.time":672}
<source>
type tail
path /var/log/foo/bar.log
pos_file /var/log/td-agent/foo-bar.log.pos
tag foo.bar
format /(?<date>[^ ]*) (?<times>[^ ]*) (?<level>[^ ]*) (([^=]+))=(?<type>([^=]+))(?:,|$) (([^=]+))=(?<name>([^=]+))(?:,|$) (([^=]+))=(?<count>([^=]+))(?:,|$)/
</source>
<source>
type tail
path /home/opdev/logs/nginx/opdev.pinpoint.korea.ncsoft.corp_access.log
pos_file /var/log/td-agent/opdev.pinpoint.access.log.pos
format /^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
time_format %d/%b/%Y:%H:%M:%S %z
tag nginx.access
</source>
<filter nginx.access>
type grep
exclude1 path css|js|png|gif|jpg|jpeg
</filter>
<match nginx.access>
type copy
<store>
type stdout
</store>
<store>
type norikra
norikra 127.0.0.1:26571
buffer_queue_limit 64
retry_limit 17
remove_tag_prefix nginx
target_map_tag true
</store>
<store>
type elasticsearch
host 127.0.0.1
port 9200
logstash_format true
type_name pinpoint
logstash_prefix nginx_access
</store>
<store>
#rabbitmq 내에서 routing key는 tag 네임으로 정의해서 bind 해서 사용해야 수집가능
type amqp
host 127.0.0.1
port 5672
user guest
password guest
exchange fluentd_exchange
exchange_type topic
exchange_durable true
payload_only false
content_type text/x-json #application/json
</store>
</match>
<source>
type norikra
norikra 127.0.0.1:26571
<fetch>
method sweep
tag query_name
tag_prefix norikra.query
interval 10s
</fetch>
</source>
<match norikra.query.access**>
type copy
<store>
type elasticsearch
host 172.20.54.152
port 9200
logstash_format true
type_name pinpoint
logstash_prefix norikra_access
</store>
<store>
type mail
host mail.host.com
port 25
from [email protected]
to [email protected]
subject SUBJECT: %s
subject_out_keys tag
out_keys tag,requests,remote
</store>
</match>
<source>
type tail
path /home/opdev/GPORTAL/logs/metrics.out
pos_file /var/log/td-agent/metrics.out.pos
format /(?<logtime>[^ ]* [^ ]*) (?<level>[^ ]*) (([^=]+))=(?<type>([^=]+))(?:,|$) (([^=]+))=(?<name>([^=]+))(?:,|$) (([^=]+))=(?<count>([^=]+))(?:,|$)/
time_format %Y-%m-%d %H:%M:%S
tag event.log
</source>
<match event.log>
type copy
<store>
type map
map ([["graphite.#{Socket.gethostname}", time, {record["name"] => record["count"].to_i}]])
multi true
</store>
</match>
<match graphite **>
type copy
<store>
type stdout
</store>
<store>
type graphite
host 127.0.0.1
port 2003
name_keys gc.ps_marksweep.count,gc.ps_marksweep.time,gc.ps_scavenge.count,gc.ps_scavenge.time,heap.committed,heap.init,heap.used,threads.daemon,threads.peak,threads.totalStarted
</store>
</match>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment