Skip to content

Instantly share code, notes, and snippets.

@siroken3
Created July 29, 2013 10:08
Show Gist options
  • Select an option

  • Save siroken3/6103354 to your computer and use it in GitHub Desktop.

Select an option

Save siroken3/6103354 to your computer and use it in GitHub Desktop.
ruby-2.0.0-p195 + fluentd 0.10.35の組み合わせでログの重複現象が確認できた fluent.conf:
## built-in TCP input
## $ echo <json> | fluent-cat <tag>
<source>
type forward
</source>
# HTTP input
# http://localhost:8888/<tag>?json=<json>
<source>
type http
port 8888
</source>
<source>
type tail
format ltsv
time_key time
path /var/log/apache2/project/project.acc.log
pos_file /var/log/fluent/log.pos
tag apache.access
</source>
<source>
type tail
format json
path /opt/xxx/fs/project/contents/app/tmp/logs/json.log
pos_file /var/log/fluent/app-log.pos
tag app.log
</source>
# Listen DRb for debug
<source>
type debug_agent
port 24230
</source>
<match apache.access>
type copy
<store>
type file
path /opt/xxx/fs/fluent/project/apache.local.acc
</store>
<store>
type forward
buffer_chunk_limit 8m
buffer_queue_limit 128
retry_limit 16
flush_interval 1s
flush_at_shutdown
buffer_type file
buffer_path /var/log/fluent/buffer/project.acc
<server>
name lg001-01
host lg001
port 24224
weight 50
</server>
# <server>
# name lg001-02
# host lg001
# port 24225
# weight 60
# </server>
# <server>
# name lg001-03
# host lg001
# port 24226
# weight 60
# </server>
# <server>
# name lg001-04
# host lg001
# port 24227
# weight 60
# </server>
<server>
name lg002-01
host lg002
port 24224
weight 50
</server>
# <server>
# name lg002-02
# host lg002
# port 24225
# weight 60
# </server>
# <server>
# name lg002-03
# host lg002
# port 24226
# weight 60
# </server>
# <server>
# name lg002-04
# host lg002
# port 24227
# weight 60
# </server>
<secondary>
type file
path /var/log/fluent/project/forward-failed/project.acc
</secondary>
</store>
</match>
<match app.log>
type forward
buffer_chunk_limit 8m
buffer_queue_limit 128
retry_limit 16
flush_interval 1s
flush_at_shutdown
buffer_type file
buffer_path /var/log/fluent/buffer/app.log
<server>
name lg001-01
host lg001
port 24224
weight 50
</server>
# <server>
# name lg001-02
# host lg001
# port 24225
# </server>
# <server>
# name lg001-03
# host lg001
# port 24226
# weight 10
# </server>
<server>
name lg002-01
host lg002
port 24224
weight 50
</server>
# <server>
# name lg002-02
# host lg002
# port 24225
# </server>
# <server>
# name lg002-03
# host lg002
# port 24226
# weight 10
# </server>
<secondary>
type file
path /var/log/fluent/project/forward-failed/app.log
</secondary>
</match>
## match tag=debug.** and dump to console
<match debug.**>
type stdout
</match>
## match not matched logs and write to file
<match **>
type file
path /var/log/fluent/else/else
</match>
@siroken3
Copy link
Author

/opt/xxx/fs/fluent/project/apache.local.acc.YYYYMMDD_#.log 個々では重複していないが、ruby 2.0.0 p195 + fluentd 0.10.35使用時に cat *.log | sort | uniq -d | wc -l で重複していることが確認できた。気になっている点は

  • apache 2.4の rotatelog -L オプションで最新のログへのハードリンクを生成させてそれを監視させている
  • rotatelogは 1日に一度のrotateなのだが fluentdのログで
    2013-07-29 19:07:10 +0900 [info]: detected rotation of /var/log/apache2/project/project.acc.log; waiting 5 seconds
    2013-07-29 19:07:10 +0900 [info]: detected rotation of /var/log/apache2/project/project.acc.log
    2013-07-29 19:07:10 +0900 [info]: following tail of /var/log/apache2/project/project.acc.log
    が不定期に発生している。そのタイミングでハッシュ付きのファイル名からログが生成されている

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment