Fluentd x Growthforecast

dstat.conf

<source>
  type dstat
  tag dstat
  option -cmln
  delay 3
</source>

# GrowthForecast用にメッセージを加工する
<match dstat>
  type copy

  # CPU関連の結果だけを取り出す
  <store>
    type map
    tag "perf.cpu"
    time time
    record record['dstat']['total cpu usage']
  </store>

  # メモリ関連の結果だけを取り出す
  <store>
    type map
    tag "perf.mem"
    time time
    record record['dstat']['memory usage']
  </store>

  <store>
    type map
    tag "perf.loadavg"
    time time
    record record['dstat']['load avg']
  </store>

  <store>
    type map
    tag "perf.network"
    time time
    record record['dstat']['net/total']
  </store>
</match>

<match perf.**>
  type forest
  subtype growthforecast

  remove_prefix perf

  <template>
    gfapi_url http://localhost:5125/api/
    service host_${hostname}
    tag_for section
    name_key_pattern .*
  </template>
</match>

gems

% sudo /usr/lib/fluent/ruby/bin/fluent-gem list                                                                                                                                                                                      [/etc/td-agent/plugin]
[sudo] password for kazu634:

*** LOCAL GEMS ***

aws-sdk (1.38.0)
bigdecimal (1.1.0)
bson (1.9.2, 1.8.6)
bson_ext (1.8.6)
bundler (1.3.6)
cool.io (1.1.1)
fluent-logger (0.4.7)
fluent-mixin-config-placeholders (0.2.4)
fluent-mixin-plaintextformatter (0.2.6)
fluent-mixin-rewrite-tag-name (0.1.0)
fluent-plugin-amplifier-filter (0.1.5)
fluent-plugin-datacounter (0.4.3)
fluent-plugin-dstat (0.2.5)
fluent-plugin-flume (0.1.1)
fluent-plugin-forest (0.2.4)
fluent-plugin-growthforecast (0.2.7)
fluent-plugin-map (0.0.4)
fluent-plugin-mongo (0.7.3)
fluent-plugin-rewrite-tag-filter (1.4.1)
fluent-plugin-s3 (0.3.7)
fluent-plugin-scribe (0.10.10)
fluent-plugin-td (0.10.18)
fluent-plugin-td-monitoring (0.1.1)
fluent-plugin-webhdfs (0.2.2)
fluentd (0.10.45)
git (1.2.6)
hirb (0.7.1)
http_parser.rb (0.5.1)
httpclient (2.3.4.1)
io-console (0.3)
iobuffer (1.1.2)
ipaddress (0.8.0)
jeweler (1.6.2)
json (1.7.7, 1.5.5)
ltsv (0.1.0)
mini_portile (0.5.3)
minitest (2.5.1)
mixlib-cli (1.4.0)
mixlib-config (2.1.0)
mixlib-log (1.6.0)
mixlib-shellout (1.3.0)
mongo (1.8.6)
msgpack (0.4.7)
nokogiri (1.5.10)
ohai (6.20.0)
parallel (0.6.5)
rake (0.9.2.2)
rdoc (3.9.5)
resolve-hostname (0.0.4)
rubyzip (0.9.9)
sigdump (0.2.2)
string-scrub (0.0.3)
systemu (2.5.2)
td (0.10.99)
td-client (0.8.58)
td-logger (0.3.23)
thrift (0.8.0)
uuidtools (2.1.4)
webhdfs (0.5.5)
yajl-ruby (1.1.0)

growthforecast.conf

####
## Output descriptions:
##

# Treasure Data (http://www.treasure-data.com/) provides cloud based data
# analytics platform, which easily stores and processes data from td-agent.
# FREE plan is also provided.
# @see http://docs.fluentd.org/articles/http-to-td
#
# This section matches events whose tag is td.DATABASE.TABLE

<source>
  type tail
  format ltsv
  tag nginx.access
  path /var/log/nginx/growthforecast.access.log
  pos_file /var/log/td-agent/buffer/access.log.pos
  time_format %d/%b/%Y:%H:%M:%S %z
</source>

<match nginx.access>
  type rewrite_tag_filter

  rewriterule1 ua (check_http|monit|Wordpress|RSS) clear
  rewriterule2 vhost ^(.*)$ site.$1
</match>

<match site.**>
  type copy

  <store>
    type file
    path /tmp/sample.log
  </store>

  <store>
    type amplifier_filter
    ratio 1000
    add_prefix amplified
    key_names taken_sec,backend_runtime
  </store>

  <store>
    type forest
    subtype datacounter
    <template>
      unit minute
      tag nginx.status.${tag}
      count_key status
      outcast_unmatched yes
      pattern1 3xx ^3\d\d$
      pattern2 4xx ^4\d\d$
      pattern3 5xx ^5\d\d$
      pattern4 200 ^200$
    </template>
  </store>
</match>

<match nginx.status.**>
  type copy

  <store>
    type growthforecast
    gfapi_url http://localhost:5125/api/
    section status_codes
    tag_for service
    remove_prefix nginx.status
    name_key_pattern .*_percentage
  </store>

  <store>
    type file
    path /tmp/foo.log
  </store>
</match>

<match amplified.**>
  type copy

  <store>
    type growthforecast
    gfapi_url http://localhost:5125/api/
    section   response
    name_keys taken_sec,backend_runtime
    tag_for service

    remove_prefix amplified
  </store>
</match>

<match clear>
  type null
</match>

td-agent.conf

include conf.d/*.conf