Custom Flume NG Agent INIT script for centos for runnig multiple agents on same machine

flume-init.d.sh

#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Starts a Flume agent
#
### BEGIN INIT INFO
# Provides:          flume
# Required-Start:    $remote_fs
# Should-Start:
# Required-Stop:     $remote_fs
# Should-Stop:
# Default-Start:     3 4 5
# Default-Stop:      0 1 2 6
# Short-Description: Flume agent
### END INIT INFO

. /lib/lsb/init-functions

# Edit /etc/default/flume to change this.
FLUME_USER=flume
FLUME_AGENT_NAME=agent
FLUME_LOG_FILE=/var/log/flume.log
FLUME_CONF_DIR=/etc/flume/conf.d
FLUME_CONF_FILE=agent.conf
if [ -f /etc/default/flume ]; then
    . /etc/default/flume
fi

if [ ! $JAVA_HOME ]; then
  export JAVA_HOME=$(readlink --canonicalize $(which java) | sed "s:bin/java::")
fi

STATUS_RUNNING=0
STATUS_DEAD=1
STATUS_DEAD_AND_LOCK=2
STATUS_NOT_RUNNING=3
ERROR_PROGRAM_NOT_INSTALLED=5

LOCKFILE="/var/lock/flume"
desc="flume"

EXEC_PATH=/usr/bin/flume-ng
FLUME_PID_FILE=/var/run/flume.pid
FLUME_SHUTDOWN_TIMEOUT=${FLUME_SHUTDOWN_TIMEOUT:-60}

start() {
  [ -x $exec ] || exit $ERROR_PROGRAM_NOT_INSTALLED

  checkstatus
  status=$?
  if [ "$status" -eq "$STATUS_RUNNING" ]; then
    exit 0
  fi

  log_success_msg "Starting $desc (flume-ng-agent): "
  touch $FLUME_PID_FILE
  chown $FLUME_USER $FLUME_PID_FILE
  /bin/su -s /bin/bash -c "/bin/bash -c 'echo \$\$ > ${FLUME_PID_FILE} && exec ${EXEC_PATH} agent --conf $FLUME_CONF_DIR --conf-file $FLUME_CONF_FILE --name $FLUME_AGENT_NAME >>${FLUME_LOG_FILE} 2>&1 ' &" $FLUME_USER
  [ $? -eq 0 ] && touch $LOCKFILE
}

stop() {
  if [ ! -e $FLUME_PID_FILE ]; then
    log_failure_msg "Flume agent is not running"
    exit 0
  fi

  log_success_msg "Stopping $desc (flume-ng-agent): "

  FLUME_PID=`cat $FLUME_PID_FILE`
  if [ -n $FLUME_PID ]; then
    kill -TERM ${FLUME_PID} &>/dev/null
    for i in `seq 1 ${FLUME_SHUTDOWN_TIMEOUT}` ; do
      kill -0 ${FLUME_PID} &>/dev/null || break
      sleep 1
    done
    kill -KILL ${FLUME_PID} &>/dev/null
  fi
  rm -f $LOCKFILE $FLUME_PID_FILE
}

restart() {
  stop
  start
}

checkstatus() {
  pidofproc -p $FLUME_PID_FILE java > /dev/null
  status=$?

  case "$status" in
    $STATUS_RUNNING)
      log_success_msg "Flume agent is running"
      ;;
    $STATUS_DEAD)
      log_failure_msg "Flume agent is dead and pid file exists"
      ;;
    $STATUS_DEAD_AND_LOCK)
      log_failure_msg "Flume agent is dead and lock file exists"
      ;;
    $STATUS_NOT_RUNNING)
      log_failure_msg "Flume agent is not running"
      ;;
    *)
      log_failure_msg "Flume agent status is unknown"
      ;;
  esac
  return $status
}

case "$1" in
  start)
    start
    ;;
  stop)
    stop
    ;;
  restart)
    restart
    ;;
  status)
    checkstatus
    ;;
  *)
    echo $"Usage: $0 {start|stop|restart|status}"
    exit 1
esac

exit 0

usage.md

This script can be used as the upstart script (in /etc/init.d) for a Flume-ng agent. We're using Flume version 1.6.

Here's the commands I'll use to set flume up on a box. What is does:

• download flume
• put the binary in /usr/bin
• put the lib directory in /usr/lib
• create a system user
• put the config files into place (see below)

cd /tmp
wget http://apache.osuosl.org/flume/1.6.0/apache-flume-1.6.0-bin.tar.gz
tar xzf apache-flume-1.6.0-bin.tar.gz
sudo mv apache-flume-1.6.0-bin/bin/flume-ng /usr/bin/flume-ng
sudo mv apache-flume-1.6.0-bin/lib /usr/lib/flume
rm -r apache-flume-1.6.0-bin*

sudo useradd --system --no-create-home flume
sudo cp ~/.../etc_defaults_flume /etc/default/flume
sudo mkdir -p /etc/flume/conf.d/
sudo cp ~/.../config/* /etc/flume/conf.d/
sudo cp ~/.../flume_init.d.sh /etc/init.d/flume
sudo chmod +x /etc/init.d/flume
sudo update-rc.d flume defaults

The config files:

• /etc/flume/conf.d/flume.conf This is where your sources, channels and sinks are defined.
• /etc/flume/conf.d/flume-env.sh Example:

JAVA_OPTS="-Xmx512m"
#FLUME_JAVA_OPTS=
FLUME_CLASSPATH=/usr/lib/flume/*
#FLUME_JAVA_LIBRARY_PATH=
#FLUME_APPLICATION_CLASS=

• /etc/flume/conf.d/log4j.properties The example can be found in the conf directory of apache-flume-1.6.0-bin.tar.gz.
• /etc/default/flume Example:

############################
# Default settings for flume
############################

# The linux user that the daemon runs as.
# Default: flume
#FLUME_USER=flume

# Name of agent that will run. This must be defined in the given conf file.
# Default: agent
#FLUME_AGENT_NAME=agent

# Path to which log file is written
# Default: /var/log/flume.log
#FLUME_LOG_FILE=/var/log/flume.log

# Location of conf files
# Default: /etc/flume/conf.d
#FLUME_CONF_DIR=/etc/flume/conf.d

# Filename of used conf file
# Default: agent.conf
#FLUME_CONF_FILE=agent.conf

good instructions

Very useful, thanks!

If you are running against 1.7.0 you need to specify full path to file like this:

FLUME_CONF_FILE=/etc/flume/conf.d/agent.conf

otherwise, it tries to look for it in current dir.