Send slack alert if Docker logs on Ubuntu server indicate NTP drifting has occurred.

ntp_drift_slack_alert.sh

#!/bin/bash

# Send a slack notification if docker logs indicate an NTP time drift
#  -- Cole Hocking

# Should be run in a cron job that runs every 24 hours

# Grab the docker container ID dynamically; in case the ID changes
#   Update: This can be statically assigned to the container NAME: 'IMAGE_NAME'
# This also assumes only one docker container is running!!
# CONTAINER_ID=$(docker ps | tr -s " " | cut -d " " -f1 | grep -v CONTAINER)

# The number of 'server time' related errors
# TODO - Replace with docker image name
NUM_TIME_ERRORS=$(docker logs --since 24h IMAGE_NAME  2>&1 | grep -c "server time" )

#IPv4 address of the server
# TODO - verify eth0 iface!
IPV4=$(ifconfig eth0 | grep "inet addr" | tr -s " " | cut -d " " -f3 | cut -d ":" -f2)

slack_message() {
    # Check if ntp service has given a timedrift output
    if [[ -f /var/lib/ntp/ntp.drift ]]; then
        NTP_DRIFT=$(cat /var/lib/ntp/ntp.drift)
    else
        NTP_DRIFT=0
    fi
    # PPM = 'Parts per million' the unit of measurement that /var/lib/ntp/ntp.drift uses. 10 ppm is about 1 second of fluctuation per day
    # This is the file that Ubuntu servers use; In RHEL/CentOS; it is located at /var/lib/ntp/drift
    echo -e "Server @ \`$IPV4\` has had \`$NUM_TIME_ERRORS\` potentially NTP-related errors in 24 hours. \\nNTP time-drift currently @ \`$NTP_DRIFT\` ppm. \\nTry \`sudo /etc/init.d/ntp restart\` to fix potential time-drift, or full reboot of \`$IPV4\` if problem is not resolved."
}

send_slack_alert(){
    # Define Slack variables
    # TODO - Modify slack webhook variables here
    CHANNEL='#channel'
    USERNAME='Webhook-Username'
    MESSAGE=$(slack_message)
    PAYLOAD="payload={\"channel\": \"$CHANNEL\", \"username\": \"$USERNAME\", \"text\": \"$MESSAGE\"}"
    HOOK='https://hooks.slack.com/services/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'

    #
    curl -X POST --data-urlencode "$PAYLOAD" "$HOOK"
}

main() {
    if [[ NUM_TIME_ERRORS -gt 5 ]]; then
        send_slack_alert
    fi
}

main