Supervision Light via bash crontab.

Readme.md

Completez le fichier supervision.cfg

Personnalisez les scripts:

• BOOT_STATUS.sh
• SEND_BOOT_STATUS.sh
• START_WATCH.sh
• WATCH.sh

Script de supervision des logs à lancer au redémarrage dans crontab

# Surveille les logs. 
@reboot sleep 30 && /home/user/supervision/START_WATCH.sh > /dev/null 2>1&
# Signale le reboot
@reboot sleep 300 && /home/user/supervision/SEND_BOOT_STATUS.sh > /dev/null 2>&1

A vous de créer le script de notification pour le boot et les logs applicatifs (ici appelé SEND_MAIL.sh dans SEND_BOOT_STATUS.sh et WATCH.sh)

BOOT_STATUS.sh

#!/bin/bash
# Author: diyfr
CONFIG_FILE=/home/user/supervision/supervision.cfg

# check container : sample check_container proxy_1
# Return container status, IP
#   OK - running
#   WARNING - restarting
#   CRITICAL - stopped
#   UNKNOWN - does not exist

check_container(){
CONTAINER=$1

if [ "x${CONTAINER}" == "x" ]; then
  echo "UNKNOWN" # - Container ID or Friendly Name Required"
  exit 3
fi
if [ "x$(which docker)" == "x" ]; then
  echo "UNKNOWN" # - Missing docker binary"
  exit 3
fi
docker info > /dev/null 2>&1
if [ $? -ne 0 ]; then
  echo "UNKNOWN" # - Unable to talk to the docker daemon"
  exit 3
fi
RUNNING=$(docker inspect --format="{{.State.Running}}" $CONTAINER 2> /dev/null)
if [ $? -eq 1 ]; then
  echo "UNKNOWN" # - $CONTAINER does not exist."
  exit 3
fi
if [ "$RUNNING" == "false" ]; then
  echo "CRITICAL" # - $CONTAINER is not running."
  exit 2
fi
RESTARTING=$(docker inspect --format="{{.State.Restarting}}" $CONTAINER)
if [ "$RESTARTING" == "true" ]; then
  echo "WARNING" # - $CONTAINER state is restarting."
  exit 1
fi
STARTED=$(docker inspect --format="{{.State.StartedAt}}" $CONTAINER)
echo "OK - started at: ${STARTED:0:19}"
}
listContainers()
{
  found=false
  while read line
  do
    if [[ $found == false && "${line:0:1}" != '[' ]]
    then
        continue
    fi
    if [[ $found == true && "${line:0:1}" = '[' ]]
    then
                if [[ -v container ]]
                then
                        printf " - %-.25s %s\n" "$name .......................... " ": `check_container $container`"
                fi
                unset name
                unset container
        continue
    fi
    found=true
    if [[ $found == true && "${line%=*}" = "name" ]]
    then
        name=${line#*=}
    fi
    if [[ $found == true && "${line%=*}" = "container" ]]
    then
        container=${line#*=}
    fi
  done
if [ -v container ]
then
        printf " - %-.25s %s\n" "$name .......................... " ": `check_container $container`"
fi
}


# Basic info
HOSTNAME=`uname -n`
# Last reboot
BOOT_EVENT=`last reboot | head -1`
#ROOT=`df -Ph | grep xvda1 | awk '{print $4}' | tr -d '\n'`
VOLUMES=`df -hk /home/docker/vol  | tail -1 | awk '{print $5}'`
DOCKER=`df -hk /home/docker/lib  | tail -1 | awk '{print $5}'`
diskusage=($(df -H | grep -vE '^Filesystem|tmpfs|cdrom|mmcblk0p1' | awk '{ print $5 " " $1 }'| cut -f1 -d '%'))
tolerance=(85)
CRITICAL_VOLUMES=""
for i in "${diskusage[@]}"
do
   if [[ "$i" -gt "$tolerance" ]]
        then
        line=$(df -hl |grep $i%)
        CRITICAL_VOLUMES="${CRITICAL_VOLUMES}
        ${line}"
   fi
done

# System load
MEMORY1=`free -t -m | grep Total | awk '{print $3" MB";}'`
MEMORY2=`free -t -m | grep "Mem" | awk '{print $2" MB";}'`
LOAD1=`cat /proc/loadavg | awk {'print $1'}`
LOAD5=`cat /proc/loadavg | awk {'print $2'}`
LOAD15=`cat /proc/loadavg | awk {'print $3'}`
echo -e "
===============================================

 - Hostname..............:  $HOSTNAME
 - Evenement boot .......:  $BOOT_EVENT
 - 'volumes' disk usage..:  $VOLUMES
 - 'docker' disk usage...:  $DOCKER
 - Volumes critiques ....:  $CRITICAL_VOLUMES

===============================================

 - CPU usage...........:  $LOAD1, $LOAD5, $LOAD15  (1, 5, 15 min)
 - Memory used.........:  $MEMORY1 / $MEMORY2
 - Swap in use.........:  `free -m | tail -n 1 | awk '{print $3}'`  MB

===============================================

 - Docker containers running

`(listContainers < $CONFIG_FILE)`

===============================================
"

SEND_BOOT_STATUS.sh

#!/bin/bash
# Author: diyfr
# PARAMETREZ LE CHEMIN DU FICHIER DE CONF $5, L'EMAIL $6 & LE SCRIPT DE NOTIFICATION LIGNE 10

CONTENT=$(/home/user/supervision/BOOT_STATUS.sh)
EMAIL="user@domain.tld"
HOSTNAME=`uname -n`
TITLE="[${HOSTNAME}] a été redémarré"
# On appelle un script qui exécute la notification à l'utilisateur
/home/user/supervision/SEND_MAIL.sh "${EMAIL}" "${TITLE}" "$CONTENT"

START_WATCH.sh

#!/bin/bash
# Author: diyfr
# PARAMETREZ LES CHEMINS DES FICHIERS ($4 à $6)
CONFIG_FILE=/home/user/supervision/supervision.cfg
SCRIPT=/home/user/supervision/WATCH.sh
PID_LIST=/home/user/supervision/watchs.pid

loadConfig()
{
  found=false
  while read line
  do
    if [[ $found == false && "${line:0:1}" != '[' ]]
    then
        continue
    fi
    if [[ $found == true && "${line:0:1}" = '[' ]]
    then
        if [[ -n ${name} && -n ${log} && -n ${criteria} && -n ${emailTo} ]]
        then
                echo $name
                /usr/bin/nohup "${SCRIPT}" "${name}" "${log}" "${criteria}" "${emailTo}" &
        fi
        unset name
        unset log
        unset criteria
        unset emailTo
        continue
    fi
    found=true
    if [[ $found == true && "${line%=*}" = "name" ]]
    then
        name=${line#*=}
    fi
    if [[ $found == true && "${line%=*}" = "log" ]]
    then
        log=${line#*=}
    fi
    if [[ $found == true && "${line%=*}" = "criteria" ]]
    then
        criteria=${line#*=}
        fi
    if [[ $found == true && "${line%=*}" = "emailTo" ]]
    then
        emailTo=${line#*=}
    fi
 done
 if [[ -n ${name} && -n ${log} &&-n ${criteria} && -n ${emailTo} ]]
 then
        echo $name
        /usr/bin/nohup "${SCRIPT}" "${name}" "${log}" "${criteria}" "${emailTo}" &
 fi
}
rm nohup.out
loadConfig < $CONFIG_FILE
sleep 1
ps -eo pid,command |grep -v grep| grep -e $SCRIPT -e "tail -n 0" |awk '{print $1}' >> $PID_LIST

STOP_WATCH.sh

#!/bin/bash
# Author: diyfr
kill -9 `cat watchs.pid`
rm watchs.pid
touch watchs.pid

supervision.cfg

# FICHIER DE COJNFIGURATION POUR LA SUPERVISION
#
#       [SECTION]  -> Nom de la section
#       name -> Libellé utilisé pour nommer l'application  ** REQUIS **
#       log -> chemin complet aux log de l'application  ** NOTIFICATION **
#       criteria -> Information à rechercher dans les logs pour déclencher la notification ** NOTIFICATION **
#       emailTo -> Adresse destinatrice du mail ** NOTIFICATION **
#       container -> nom du container docker associé ** BOOT STATUS **
# [EXEMPLE]
# name=Exemple
# container=exemple-app
# [EXEMPLE 2]
# name=Exemple2
# log=/home/docker/volumes/app/exempleApp/log/exempleApp.log
# criteria=ERROR
# emailTo=toto@domain.tld
# container=exemple-app-2

WATCH.sh

#!/bin/bash
# Author: diyfr
# PARAMETREZ LE SCRIPT DE NOTIFICATION LIGNE 10

# param 1 nom de l'application
# param 2 fichier de log
# param 3 mot clé
# param 4 email

SCRIPT=/home/user/supervision/SEND_MAIL.sh
TITLE="[SUPERVISION] ${1}"
HOSTNAME=`uname -n`

tail -n 0 -F $2 | \
while read LINE
do
        echo "$LINE" | grep -q $3
        if [ $? = 0 ]
        then
                MSG="Le script de supervision sur le serveur ${HOSTNAME} a détecté un message dans les logs correspondant à vos critères.\nVoici la ligne concernée:\n${LINE}"
                $SCRIPT "${4}" "${TITLE}" "${MSG}"
        fi
done