fluentd agent

agent_fluentd.conf

<source>
  type tail
  path path_to_file1
  pos_file path_to_pos_file1
  tag app.file1
  format /^(?<ip>[^ ]+) [^ ]+ [^ ]+ \[(?<time>[^\]]+)\] "(?<method>\S+) /(?<country>[A-Z0-9]{0,3})_(?<domain>[^/]+)/(?<module>[^/]+)/(?<controller>[^/]+)/(?<action>[^/]+)/(?<record>\S*)" (?<code>\d+) (?<res_time>\d+) "(?<referer>[^\"]*)" "(?<old_device>[^_\" ]*)_(?<device>[^_\" ]*)_(?<browser>[^_\" ]*) ?(?<guid>[^\"]*)" "(?<vuid>\d*) ?(?<option>[^\"]*)"$/
  time_format %d/%b/%Y:%H:%M:%S %z
  rotate_wait = 5s
</source>

<source>
  type tail
  path path_to_file2
  pos_file path_to_pos_file1
  tag app.file1
  format /^[^ ]+ [^ ]+ [^ ]+ \[(?<time>[^\]]+)\] "\S+ /(?<module>[^/]+)/(?<controller>[^/]+)/(?<ad>[^\"]+)/" (?<code>\d+) (?<res_time>\d+) "(?<try_count>\d+)(?<result>[^\"]+)".*$/
  time_format %d/%b/%Y:%H:%M:%S %z
  rotate_wait = 5s
</source>

<source>
  type tail
  path path_to_file3
  pos_file path_to_pos_file1
  tag app.file1
  format /(?<time>[^,]+,[^,]+),(?<res_time>\d+),(?<device>[^,]+),/(?<module>[^/]+)/(?<controller>[^/]+),(?<ad>[^\"]*)$/
  time_format %Y/%m/%d,%H:%M:%S
  rotate_wait = 5s
</source>

<match app.*>
  type roundrobin
  <store>
    type forward
    buffer_type memory
    buffer_chunk_limit 256k
    buffer_queue_limit 16
    flush_interval 1s
    retry_limit 5
    retry_wait 1s
    send_timeout 5s
    recover_wait 5s
    heartbeat_interval 1s
    phi_threshold 10
    hard_timeout 5s
    <server>
      name worker1
      host 192.168.xxx.xxx
      port 24224
      weight 10
    </server>
    <server>
      name worker2
      host 192.168.xxx.xxx
      port 24224
      weight 10
    </server>
    ...
  </store>
  <store>
    ...
  </store>
  ...
</match>

fluentd.sh

#!/bin/bash

FLUENTD_HOME=/data/App/fluentd
PID_FILE=$FLUENTD_HOME/fluentd.pid
CONF_FILE=$FLUENTD_HOME/conf/fluentd.conf
LOG_FILE=$FLUENTD_HOME/logs/fluentd.out
PLUGIN_DIR=$FLUENTD_HOME/plugins
PSNAME="fluentd --daemon"
F_USER=xxxx
F_GROUP=xxxx
RUBY_VER="1.9.3-p125"

start()
{
  PID=`pgrep -f "$PSNAME"`
  if [ -z "$PID" ]; then
    if [ -f $PID_FILE ]; then rm -f $PID_FILE; fi
  else
    echo "fluentd already started."
    return -1
  fi
  if [ -z $rvm_path ]; then
    export rvm_path=$FLUENTD_HOME/rvm
  fi
  source $rvm_path/scripts/rvm
  rvm use $RUBY_VER 1> /dev/null
  echo -n "Starting fluentd: "
  fluentd --daemon $PID_FILE --user $F_USER --group $F_GROUP --config $CONF_FILE --plugin $PLUGIN_DIR --log $LOG_FILE -vv
  echo "done."
}

stop()
{
  PID=`pgrep -f "$PSNAME"`
  if [ -z "$PID" ]; then
    echo "fluentd already stopped."
    return 0
  fi
  echo -n "Stopping fluentd: "
  pkill -TERM -f "$PSNAME"
  count=0
  while [ ! -z "$PID" ]; do
    count=`expr $count + 1`
    if [ `expr $count % 10` == 0 ]; then pkill -KILL -f "$PSNAME"; fi
    if [ $count == 60 ]; then
      echo " failed."
      return -1
    fi
    sleep 1
    PID=`pgrep -f "$PSNAME"`
  done
  rm -f $PID_FILE
  echo "done."
}

restart()
{
  stop && start
}

update()
{
  source $rvm_path/scripts/rvm
  rvm use $RUBY_VER 1> /dev/null
  gem update fluentd
}

case "$1" in
  start)
    start
    ;;
  stop)
    stop
    ;;
  restart)
    restart
    ;;
  reload)
    pkill -HUP -f "$PSNAME"
    ;;
  condrestart)
    if [ ! -f $PID_FILE ]; then
      restart
    elif [ ! -f /proc/`cat $PID_FILE`/status ]; then
      restart
    fi
    ;;
  update)
    stop
    update
    start
    ;;
  *)
    echo "Usage: $0 {start|stop|restart|reload}"
    exit 1
    ;;
esac
exit $?

setup.sh

#!/bin/bash

if [ -d /data/App/fluentd/rvm/rubies ]; then
  echo "/data/App/fluentd/rvm/rubies already exists."
  exit -1
fi

mkdir /data/App/fluentd/rvm -p
export rvm_path=/data/App/fluentd/rvm
if [ -z "`grep '^insecure$' ~/.curlrc`" ]; then
  echo insecure >> ~/.curlrc
fi
bash -s stable < <(curl -s https://raw.github.com/wayneeseguin/rvm/master/binscripts/rvm-installer)
sed -ie '/Add RVM/d' ~/.bashrc
sed -ie '/Load RVM/d' ~/.bash_profile
echo "export rvm_path=/data/App/fluentd/rvm" >> ~/.bash_profile
echo "[[ -s \"\$rvm_path/scripts/rvm\" ]] && source \"\$rvm_path/scripts/rvm\"" >> ~/.bash_profile
echo >> ~/.bash_profile
source ~/.bash_profile
rvm install 1.9.3-p125
rvm use 1.9.3-p125
gem update --system
gem install fluentd