Docker Fluentd Flowcounter Statsd (or Graphite) example

Docker Fluentd with FlowCounter and Statsd.md

This is an example of making fluentd count log messages and emit to graphite.

Run in a terminal:

# start graphite statsd
docker run -d --name graphite --restart=always -p 80:80 -p 2003-2004:2003-2004 -p 2023-2024:2023-2024 -p 8125:8125/udp -p 8126:8126 hopsoft/graphite-statsd
# start fluentd container
./build.sh

Then in another terminal session:

docker run --name source4 --log-driver=fluentd --log-opt tag="docker.socool.{{.ID}}" --log-opt fluentd-address="$(docker inspect -f '{{.NetworkSettings.IPAddress}}' custom-docker-fluent-logger):24224" python:alpine echo $'Hello\nBye\nMore\nAnd\n\So\nLots'

build.sh

#!/bin/bash
#From https://hub.docker.com/r/fluent/fluentd/
#curl https://raw.githubusercontent.com/fluent/fluentd-docker-image/master/v0.14/alpine-onbuild/fluent.conf > fluent.conf
mkdir -p log
mkdir -p mkdir plugins
cd plugins
#gem fetch fluent-plugin-flowcounter
#gem fetch fluent-plugin-graphite
#gem fetch fluent-plugin-graphite
cd ..
docker build -t custom-fluentd:latest ./

docker run -it --rm \
    --name custom-docker-fluent-logger \
    -v "$(pwd)/log:/fluentd/log" --net=host custom-fluentd:latest

#docker inspect -f '{{.NetworkSettings.IPAddress}}' custom-docker-fluent-logger
#docker run --log-driver=fluentd --log-opt tag="docker.{{.ID}}" --log-opt fluentd-address="$(docker inspect -f '{{.NetworkSettings.IPAddress}}' custom-docker-fluent-logger):24224" python:alpine echo Hello
#docker kill -s USR1 custom-docker-fluent-logger

# FLUENT_ID=$(docker ps --format '{{.ID}}')
FLUENT_ID=$(docker inspect -f '{{.ID}}' custom-docker-fluent-logger)
# Inspect logs, tweak config
#docker exec -t -i $(docker inspect -f '{{.ID}}' custom-docker-fluent-logger) sh
#docker exec -t -i $(docker inspect -f '{{.ID}}' graphite) sh
#docker cp fluent.conf ${FLUENT_ID}:/fluentd/etc/
#docker exec ${FLUENT_ID} bash -c 'cd /opt/bash2py-3.5 ; ./bash2py build.sh'
#docker cp ${FLUENT_ID}:/opt/bash2py-3.5/build.sh.py build.py

Dockerfile

FROM fluent/fluentd:v0.14

MAINTAINER Steve Coffman <[email protected]>

WORKDIR /home/fluent

ENV PATH /home/fluent/.gem/ruby/2.3.0/bin:$PATH

# Run as root for backwards compatibility
USER root

#    gem install fluent-plugin-retag:0.0.1 && \
#    gem install fluent-plugin-route:1.0.0 && \
# Do not split this into multiple RUN!
# Docker creates a layer for every RUN-Statement
# therefore an 'apk delete build*' has no effect
RUN apk --no-cache --update add \
                            build-base \
                            ruby-dev \
                            libstdc++ && \
    gem install fluent-plugin-flowcounter -v 0.4.2 && \
    gem install statsd-ruby -v 1.4.0 && \
    gem install fluent-plugin-statsd -v 1.0.2 && \
    gem install fluent-plugin-graphite:0.0.6 && \
    gem install fluent-plugin-record-reformer -v 0.9.0 && \
    apk del build-base ruby-dev && \
    rm -rf /tmp/* /var/tmp/* /var/cache/apk/* /usr/lib/ruby/gems/*/cache/*.gem && \
    chown -R fluent:fluent /fluentd
COPY fluent.conf /fluentd/etc/$FLUENTD_CONF
EXPOSE 24220 24224

CMD fluentd -c /fluentd/etc/$FLUENTD_CONF -p /fluentd/plugins $FLUENTD_OPT

fluent.conf

<source>
  @type forward
  @id  input1
  @label @raw
  port  24224
</source>

<filter fluent.**>
  @type stdout
</filter>

<label @raw>
  <filter docker.**>
  @type stdout
  </filter>
  <match docker.**>
      @type flowcounter
      #Change this label to flowcount_graphite to route there
      @label @flowcount_statsd
      # This is the name of the fluentd hostname, not source
      tag flowcount.${hostname}
      aggregate tag
      output_style tagged
      count_keys log
      # unit second fine for statsd
      # one second interval will eat repeated graphite stats
      unit minute
      # Do not re-emit metrics if no new received
      delete_idle true
  </match>
</label>

<label @flowcount_graphite>
  <filter **>
    @type stdout
  </filter>
  #
  # Match messages that came from the "flowcounter" plugin and duplicate them.
  # The resulting messages should have different label though, so that we can
  # process them separately.
  #
  <match **>
    @type record_reformer
    tag ${tag + ".count"}
    @label @graphite
  </match>
</label>

<label @flowcount_statsd>
  <filter **>
    @type stdout
  </filter>
  #
  # Match messages that came from the "flowcounter" plugin and duplicate them.
  # The resulting messages should have different label though, so that we can
  # process them separately.
  #
  <match **>
    @type record_reformer
    @label @statsd
    tag statsd.${tag}
    statsd_key ${tag}
    statsd_count ${count}
    statsd_type ${"count"}
    remove_keys count,bytes,count_rate,bytes_rate,tag
  </match>
</label>

<label @statsd>
  <filter **>
    @type stdout
  </filter>
  <match **>
    @type copy
    <store>
      @type statsd
      host 127.0.0.1
      port 8125
      flush_interval 1s
    </store>
    <store>
      @type stdout
    </store>
  </match>
</label>

<label @graphite>
  <filter **>
    @type stdout
  </filter>
  <match **>
    @type copy
    <store>
      @type graphite
      host localhost
      port 2003
      tag_for prefix
      name_keys count
    </store>
    <store>
      @type stdout
    </store>
  </match>
</label>

<match **>
  @type file
  @id   output1
  path         /fluentd/log/data.*.log
  symlink_path /fluentd/log/data.log
  append       true
  time_slice_format %Y%m%d
  time_slice_wait   10m
  time_format       %Y%m%dT%H%M%S%z
</match>

make_graphite.sh

#!/bin/bash

# This image will run local graphite and statsd in a container
# See here: https://github.com/hopsoft/docker-graphite-statsd
docker run -d\
  --name graphite\
  --restart=always\
  -p 80:80\
  -p 2003-2004:2003-2004\
  -p 2023-2024:2023-2024\
  -p 8125:8125/udp\
  -p 8126:8126\
  hopsoft/graphite-statsd

# docker exec -t -i $(docker inspect -f '{{.ID}}' graphite) bash
# docker kill -s USR1 graphite