primary.conf

<source>
  type forward
  port 24224
</source>

<match **>
  type forward
  <server>
    host test-server
  </server>
  buffer_type file
  buffer_path /path/to/buffer
  flush_interval 1s
</match>

secondary.conf

<source>
  type forward
  port 24224
</source>

<match **>
  type forward
  <server>
    host test-server
  </server>
  <secondary>
    type s3
    aws_key_id  xxxxxx
    aws_sec_key yyyyyy
    s3_bucket   your_bucket
    s3_region ap-northeast-1
    s3_object_key_format %{path}.%{file_extension}
    path nginx

    buffer_path /path/to/s3_forward
    time_slice_format %Y%m%d
    time_slice_wait 5m
    utc
  </secondary>
  buffer_type file
  buffer_path /path/to/buffer
  flush_interval 1s
  retry_limit 5
  max_retry_wait 10s
</match>

shell.md

file buffer

$ echo '{"foo": "bar"}' | fluent-cat debug.test

$ ruby unpack.rb /path/to/forward.debug.test.xxx.log
[1459155557, {"foo"=>"bar"}]

s3 buffer

$ echo '{"foo": "bar"}' | fluent-cat debug.test
$ aws s3 cp s3://path/to/buffer/forward_debug.test.gz .
$ gunzip forward_debug.test.gz

$ ruby unpack.rb /path/to/forward.debug.test
[1459155737, {"foo"=>"bar"}]

unpack.rb

require 'msgpack'
p MessagePack.unpack(File.open(ARGV[0]).read)