fluentd configuration file for LTSV format log file

gistfile1.txt

####
## Output descriptions:
##

# Treasure Data (http://www.treasure-data.com/) provides cloud based data
# analytics platform, which easily stores and processes data from td-agent.
# FREE plan is also provided.
# @see http://docs.fluentd.org/articles/http-to-td
#
# This section matches events whose tag is td.DATABASE.TABLE

<source>
  type tail
  format ltsv
  tag nginx.access
  path /var/log/nginx/growthforecast.access.log
  pos_file /var/log/td-agent/buffer/access.log.pos
  time_format %d/%b/%Y:%H:%M:%S %z
</source>

<match nginx.access>
  type rewrite_tag_filter

  rewriterule1 ua (check_http|monit|Wordpress|RSS) clear
  rewriterule2 vhost ^(.*)$ site.$1
</match>

<match site.**>
  type copy

  <store>
    type file
    path /tmp/sample.log
  </store>

  <store>
    type amplifier_filter
    ratio 1000
    add_prefix amplified
    key_names taken_sec,backend_runtime
  </store>

  <store>
    type forest
    subtype datacounter
    <template>
      unit minute
      tag nginx.status.${tag}
      count_key status
      outcast_unmatched yes
      pattern1 3xx ^3\d\d$
      pattern2 4xx ^4\d\d$
      pattern3 5xx ^5\d\d$
      pattern4 200 ^200$
    </template>
  </store>
</match>

<match nginx.status.**>
  type copy

  <store>
    type growthforecast
    gfapi_url http://localhost:5125/api/
    section status_codes
    tag_for service
    remove_prefix nginx.status
    name_key_pattern .*_percentage
  </store>

  <store>
    type file
    path /tmp/foo.log
  </store>
</match>

<match amplified.**>
  type copy

  <store>
    type growthforecast
    gfapi_url http://localhost:5125/api/
    section   response
    name_keys taken_sec,backend_runtime
    tag_for service

    remove_prefix amplified
  </store>
</match>

<match clear>
  type null
</match>