Logstash configuration example

init.d-logstash.sh

#!/bin/bash
#
# /etc/init.d/logstash -- startup script for LogStash.
#
### BEGIN INIT INFO
# Provides:          logstash
# Required-Start:    $all
# Required-Stop:     $all
# Should-Start:      $network $syslog
# Should-Stop:       $network $syslog
# Default-Start:     2 3 4 5
# Default-Stop:      0 1 6
# Short-Description: Starts logstash
# Description:       Starts logstash using start-stop-daemon
### END INIT INFO

set -e

PATH=/bin:/usr/bin:/sbin:/usr/sbin
NAME=logstash
DESC="Logstash Daemon"
DEFAULT=/etc/default/$NAME
START=yes

if [ `id -u` -ne 0 ]; then
   echo "You need root privileges to run this script"
   exit 1
fi

. /lib/lsb/init-functions

if [ -r /etc/default/rcS ]; then
   . /etc/default/rcS
fi

# The following variables can be overwritten in $DEFAULT

# Run logstash as this user ID and group ID
LS_USER=logs
LS_GROUP=logs

JAVA=/usr/bin/java

# Directory where the logstash all in one jar lives
LS_HOME=/opt/logs/logstash

# Additional Java OPTS
LS_JAVA_OPTS=" -Djava.io.tmpdir=$LS_HOME/tmp"

# logstash log directory
LOG_DIR=/opt/logs/logstash/logs

# logstash configuration directory
CONF_DIR=/opt/logs/logstash

# logstash configuration directory
CONF_FILE=$CONF_DIR/conf/logstash.conf

# logstash log file
LOG_FILE=$LOG_DIR/$NAME.log

# Open File limit
OPEN_FILES=2048

# Nice level
NICE=19

# Filter threads
FILTER_THREADS=1

# End of variables that can be overwritten in $DEFAULT

# overwrite settings from default file
if [ -f "$DEFAULT" ]; then
   . "$DEFAULT"
fi

# Define other required variables
PID_FILE=/var/run/$NAME.pid
DAEMON=$LS_HOME/bin/logstash.jar
DAEMON_OPTS="agent -f ${CONF_FILE} --log ${LOG_FILE} -w ${FILTER_THREADS} -- web"

is_true() {
    if [ "x$1" = "xtrue" -o "x$1" = "xyes" -o "x$1" = "x1" ] ; then
        return 0
    else
        return 1
    fi
}

# Check DAEMON exists
if ! test -e $DAEMON; then
   log_failure_msg "Daemon $DAEMON doesn't exist"
   exit 1
fi

case "$1" in
   start)
      if ! is_true "$START" ; then
         echo "logstash not configured to start, please edit $DEFAULT to enable"
         exit 0
      fi

      if [ -z "$JAVA" ]; then
         log_failure_msg "no JDK found - $JAVA"
         exit 1
      fi

      # Check if a config file exists
      if ! test -e $CONF_FILE; then
         log_failure_msg "There aren't any configuration files in $CONF_DIR"
         exit 1
      fi
      
      if [ ! -d "$LOG_DIR" ]; then
         mkdir -p "$LOG_DIR"
      fi
      
      log_daemon_msg "Starting $DESC"

      if start-stop-daemon --test --start --pidfile "$PID_FILE" \
         --user "$LS_USER" --exec "$JAVA" \
      >/dev/null; then
         # Prepare environment
         ulimit -n $OPEN_FILES

         # Start Daemon
         start-stop-daemon --start -b --user "$LS_USER" -c "$LS_USER":"$LS_GROUP" \
           -d "$LS_HOME" --pidfile "$PID_FILE" --make-pidfile \
           --exec "$JAVA" -- $LS_JAVA_OPTS -jar $DAEMON $DAEMON_OPTS

         sleep 1

         if start-stop-daemon --test --start --pidfile "$PID_FILE" \
            --user "$LS_USER" --exec "$JAVA" \
         >/dev/null; then

            if [ -f "$PID_FILE" ]; then
               rm -f "$PID_FILE"
            fi

            log_end_msg 1
         else
            log_end_msg 0
         fi
      else
         log_progress_msg "(already running)"
         log_end_msg 0
      fi
   ;;
   stop)
      log_daemon_msg "Stopping $DESC"

      set +e

      if [ -f "$PID_FILE" ]; then
         start-stop-daemon --stop --pidfile "$PID_FILE" \
            --user "$LS_USER" \
            --retry=TERM/20/KILL/5 >/dev/null

         if [ $? -eq 1 ]; then
            log_progress_msg "$DESC is not running but pid file exists, cleaning up"
         elif [ $? -eq 3 ]; then
            PID="`cat $PID_FILE`"
            log_failure_msg "Failed to stop $DESC (pid $PID)"
            exit 1
         fi

         rm -f "$PID_FILE"
      else
         log_progress_msg "(not running)"
      fi

      log_end_msg 0
      set -e
   ;;
   status)
      set +e

      start-stop-daemon --test --start --pidfile "$PID_FILE" \
         --user "$LS_USER" --exec "$JAVA" \
      >/dev/null 2>&1

      if [ "$?" = "0" ]; then
         if [ -f "$PID_FILE" ]; then
            log_success_msg "$DESC is not running, but pid file exists."
            exit 1
         else
            log_success_msg "$DESC is not running."
            exit 3
         fi
      else
         log_success_msg "$DESC is running with pid `cat $PID_FILE`"
      fi

      set -e
   ;;
   restart|force-reload)
      if [ -f "$PID_FILE" ]; then
         $0 stop
         sleep 1
      fi

      $0 start
   ;;
   *)
      log_success_msg "Usage: $0 {start|stop|restart|force-reload|status}"
      exit 1
   ;;
esac

exit 0

kibana

server {
	listen   80; ## listen for ipv4; this line is default and implied
	listen   [::]:80 default_server ipv6only=on; ## listen for ipv6

	root /opt/logs/web/kibana;
	index index.html;

	# Make site accessible from http://localhost/
	server_name localhost;

	location / {
		try_files $uri /index.html =404;
	}
}

logrorate.d-logstash.sh

/opt/logs/logstash/logs/logstash.log {
    daily
    copytruncate
    rotate 14
    compress
}

logstash.conf.rb

input {
  file {
    exclude => ['*.gz']
    path => ['/opt/apps/logs/*.log']
    type => 'apps'
    #start_position => 'beginning'
  }
}


filter {
  if [type] == "PasteParser_Preprod" {

    # log.debug("{} - end in - {}".format(func.func_name, (datetime.now() - time_start).microseconds))
    grok {
      # BUG https://logstash.jira.com/browse/LOGSTASH-1316
      patterns_dir => ['/opt/logs/logstash/src/patterns']
      match => ["message", "%{DATESTAMP:date} - apps - %{LOGLEVEL:loglevel} - %{WORD:action} - end in - %{POSINT:duration:int}"]
    }

    # log.debug("search_cc - cc_found - {} - {}".format(cc, issuer))
    grok {
      patterns_dir => ['/opt/logs/logstash/src/patterns']
      match => ["message", "%{DATESTAMP:date} - apps - %{LOGLEVEL:loglevel} - action - subaction - %{POSINT:cc:int} - %{WORD:issuer}"]
      add_field => ["action", "action"]
      add_field => ["sub_action", "sub_action"]
    }

    # BUG https://logstash.jira.com/browse/LOGSTASH-1340
    # 2013-10-10 03:25:33,128
    #date {
    #  match => ["date", "yyyy-mm-dd HH:mm:ss,SSS"]
    #  locale => "fr"
    #}
    mutate {
      remove_field => [ "host" ]
      #replace => ["@timestamp", "%{date}" ]
    }
  }
}   
output {

  elasticsearch {
    embedded => true
  }
}