収集したログが少なくなっているconfファイル (Amazon EC2) 各webサーバで稼働しているfluentdから2台のlogクラスタへ転送し、最終的にbatchサーバへ集約 fluentd-web.conf (10から20台) -> fluentd-log.conf (2台) -> fluentd-bach.conf fluentd: 0.10.35 ruby: 2.0.0p195 web: c1.xlarge, apache 2.4.1 & 付属のrotatelogs -L で最新のログをproject.acc.logという名前でハードリンクしたログを in_tailで収集 lg001, lg002: webサーバから集約したログを s3, batch, mongodbへ転送

fluentd-batch.conf

## built-in TCP input
## $ echo <json> | fluent-cat <tag>
<source>
  type forward
  port 24224
</source>

# Listen DRb for debug
<source>
  type debug_agent
  port 24230
</source>

## match tag=debug.** and dump to console
<match debug.**>
  type stdout
</match>

## match access
<match access.**>
  type copy
  <store>
    type file_alternative
    path /var/log/fluent/project/%Y-%m-%d/project.01.%Y-%m-%d-%H
    buffer_chunk_limit 512m
    buffer_queue_limit 64
    time_slice_wait 10m
    flush_interval 15m
    compress gz
  </store>
  <store>
    type s3
    s3_bucket xxx-xxx-xxx
    s3_endpoint s3.amazonaws.com
    s3_object_key_format %{path}%{time_slice}/%{hostname}/project.acc.01.%{index}.%{file_extension}
    path access/
    auto_create_bucket true
    time_slice_format %Y-%m-%d-%H
    buffer_path /var/log/fluent/buffer/s3/acc.01
    buffer_chunk_limit 64m
    buffer_queue_limit 128
    flush_interval 2m
    flush_at_shutdown
    <secondary>
      type file
      path /var/log/fluent/project/forward-failed/s3/acc.01
    </secondary>
  </store>
</match>

# match not matched logs and write to file
<match **>
  type file
  path /var/log/fluent/else/else.01
  compress gz
</match>

fluentd-log.conf

## built-in TCP input
## $ echo <json> | fluent-cat <tag>
<source>
  type forward
  port 24224
</source>

# HTTP input
# http://localhost:8888/<tag>?json=<json>
<source>
  type http
  port 8888
</source>

# Listen DRb for debug
<source>
  type debug_agent
  port 24230
</source>

## match tag=debug.** and dump to console
<match debug.**>
  type stdout
</match>

## match tag=apache.access and write to file
<match apache.access>
  type rewrite_tag_filter
  rewriterule1 req ^.*main\.php access.iphone
  rewriterule2 req ^.*main_pad\.php access.ipad
  rewriterule3 req ^.*main_android\.php access.android
  rewriterule4 req ^.*main_amazon\.php access.amazon
  rewriterule5 req .+ clear
</match>

<match access.**>
  type copy
  <store>
    type forward
    buffer_type file
    buffer_path /var/log/fluent/buffer/batch02/batch02.acc.01
    buffer_queue_limit 256
    flush_interval 10s
    flush_at_shutdown
    <server>
      name batch02-01
      host batch02
      port 24224
      weight 60
    </server>
    <secondary>
      type file
      path /var/log/fluent/project/forward-failed/batch02.01
    </secondary>
  </store>
  <store>
    type s3
    s3_bucket xxx-s3-backet-name
    s3_endpoint s3.amazonaws.com
    s3_object_key_format %{path}%{time_slice}/%{hostname}/project.acc.01.%{index}.%{file_extension}
    path access/
    auto_create_bucket true
    time_slice_format %Y-%m-%d-%H
    buffer_path /var/log/fluent/buffer/s3/acc.01
    buffer_chunk_limit 64m
    buffer_queue_limit 128
    flush_interval 2m
    flush_at_shutdown
    <secondary>
      type file
      path /var/log/fluent/project/forward-failed/s3/acc.01
    </secondary>
  </store>
  <store>
    type forward
    flush_interval 10s
    buffer_type file
    buffer_path /var/log/fluent/buffer/mongodb/mongodb.acc.01
    <server>
      name mongodb-01
      host mongodb
      port 24224
      weight 25
    </server>
  </store>
</match>

<match app.log>
  type copy
  <store>
    type forward
    buffer_type file
    buffer_path /var/log/fluent/buffer/mongodb/mongodb.app.01
    flush_interval 1s
    <server>
      name mongodb-01
      host mongodb
      port 24224
      weight 25
    </server>
  </store>
  <store>
    type s3
    s3_bucket xxx-s3-backet-name
    s3_endpoint s3.amazonaws.com
    s3_object_key_format %{path}%{time_slice}/%{hostname}/app.01.%{index}.%{file_extension}
    path app.log/
    auto_create_bucket true
    time_slice_format %Y-%m-%d
    buffer_path /var/log/fluent/buffer/s3/app.01
    buffer_chunk_limit 64m
    buffer_queue_limit 128
    flush_interval 2m
    flush_at_shutdown
    <secondary>
      type file
      path /var/log/fluent/project/forward-failed/s3/app.01
    </secondary>
  </store>
</match>

<match gamelog.**>
    type forward
    retry_limit 15
    flush_interval 1s
    flush_at_shutdown
    buffer_type file
    buffer_path /var/log/fluent/buffer/game.log
    <server>
      name fluentd-01.analyzer
      host 127.0.0.1
      port 34224
      weight 60
    </server>
    <server>
      name fluentd-02.analyzer
      host 127.0.0.1
      port 34225
      weight 60
    </server>
    <secondary>
      type file
      path /var/log/fluent/project/forward-failed
    </secondary>
</match>

## match fluentd.**

## match not matched logs and write to file
<match **>
  type file
  path /var/log/fluent/else/else.01
  compress gz
</match>

fluentd-web.conf

## built-in TCP input
## $ echo <json> | fluent-cat <tag>
<source>
  type forward
</source>

# HTTP input
# http://localhost:8888/<tag>?json=<json>
<source>
  type http
  port 8888
</source>

<source>
  type tail
  format ltsv
  time_key time
  path /var/log/apache2/project/project.acc.log
  pos_file /var/log/fluent/log.pos
  tag apache.access
</source>

<source>
  type tail
  format json
  path /var/log/project/contents/app/tmp/logs/json.log
  pos_file /var/log/fluent/app-log.pos
  tag app.log
</source>

# Listen DRb for debug
<source>
  type debug_agent
  port 24230
</source>

<match apache.access>
  type forward
  buffer_chunk_limit 8m
  buffer_queue_limit 128
  retry_limit 16
  flush_interval 1s
  flush_at_shutdown
  buffer_type file
  buffer_path /var/log/fluent/buffer/project.acc
  <server>
    name lg001-01
    host lg001
    port 24224
    weight 50
  </server>
  <server>
    name lg002-01
    host lg002
    port 24224
    weight 50
  </server>
  <secondary>
    type file
    path /var/log/fluent/project/forward-failed/project.acc
  </secondary>
</match>

<match app.log>
  type forward
    buffer_chunk_limit 8m
    buffer_queue_limit 128
    retry_limit 16
    flush_interval 1s
    flush_at_shutdown
    buffer_type file
    buffer_path /var/log/fluent/buffer/app.log
    <server>
      name lg001-01
      host lg001
      port 24224
      weight 50
    </server>
    <server>
      name lg002-01
      host lg002
      port 24224
      weight 50
    </server>
    <secondary>
      type file
      path /var/log/fluent/project/forward-failed/app.log
    </secondary>
</match>

<match gamelog.**>
    type forward
    retry_limit 15
    flush_interval 1s
    flush_at_shutdown
    buffer_type file
    buffer_path /var/log/fluent/buffer/game.log
    <server>
      name lg001
      host lg001
      port 24224
      weight 60
    </server>
    <server>
      name lg002
      host lg002
      port 24224
      weight 60
    </server>
    <secondary>
      type file
      path /var/log/fluent/project/forward-failed
    </secondary>
</match>

## match tag=debug.** and dump to console
<match debug.**>
  type stdout
</match>

## match not matched logs and write to file
<match **>
  type file
  path /var/log/fluent/else/else
</match>