Fluentd some sample config used in prod

<source>
  @type tail
  path /var/log/assignment_server.out.log*
  pos_file /var/run/td-agent/td-agent_position.pos
  tag prod.allocation
  path_key tailed_path
  rotate_wait 5
  read_from_head true
  refresh_interval 60
  multiline_flush_interval 3s
  <parse>
    @type multiline
    format_firstline /\d{4}-\d{1,2}-\d{1,2}/
    format1 /^(?<log>.*)/
  </parse>
</source>

<source>
  @type tail
  path /var/lib/docker/containers/*/*.log
  pos_file /var/run/td-agent/td-agent_position.pos
  enable_stat_watcher false
  tag prod.allocation
  path_key tailed_path
  rotate_wait 5
  read_from_head true
  refresh_interval 60
  <parse>
    @type json
    time_format %Y-%m-%dT%H:%M:%S
  </parse>
</source>

<filter prod.allocation.**>
  @type record_transformer
  <record>
    hostname ${hostname}
  </record>
  remove_keys stream
</filter>

<match prod.allocation.**>
  @type "aws-elasticsearch-service"
  logstash_prefix allocation.fd
  logstash_format true
  include_tag_key true
  tag_key tag
  <endpoint>
    url https://XXXXX.es.amazonaws.com
    region ap-southeast-1
    access_key_id "XXXXX"
    secret_access_key "XXXXXX"
  </endpoint>
  <buffer>
      @type memory
      flush_mode interval
      retry_type exponential_backoff
      flush_thread_count 2
      flush_interval 60s
      retry_forever
      retry_max_interval 30
      chunk_limit_size 8M
      queued_chunks_limit_size 30
      flush_at_shutdown true
      overflow_action block
  </buffer>
</match>

<match prod.allocation.**>
  @type "aws-elasticsearch-service"
  logstash_prefix allocation.fd
  logstash_format true
  include_tag_key true
  tag_key tag
  <endpoint>
    url https://XXXXX.es.amazonaws.com
    region ap-southeast-1
    access_key_id "XXX"
    secret_access_key "XXXX"
  </endpoint>
  <buffer>
      @type file
      path /var/log/td-agent/buffers/fluentd-prod.system.buffer
      flush_mode interval
      retry_type exponential_backoff
      flush_thread_count 2
      flush_interval 60s
      retry_forever
      retry_max_interval 30
      chunk_limit_size 100M
  </buffer>
</match>

<match prod.allocation.**>
  @type stdout
</match>

rahulwa/fluentd-prod-sample-config.md