EdgeOS ready script that tests for WAN connection degradation (via packet loss and latency checks)

network_test.sh

#!/bin/sh
#
# USG and EdgeRouter route-test script for failover
#
# uses ping to check latency and packet loss.
#  returns zero when within thresholds and non zero (1) when above thresholds
#  uses ping, traceroute, and logger
#
# by https://github.com/popmonkey
#
# v0.8.1 2019.10.12 - don't wait for log_traceroute to return failure
# v0.8.0 2019.10.06 - on failure, dump traceroute info
# v0.7.0 2019.10.05 - ping more than one host
# v0.6.2 2019.10.04 - refactored - also checks packetloss
# v0.5.0 2019.10.03 - first working version
#
# this is meant to be used as the test script in the load-balance groups
# e.g.
#  group wan_failover {
#      interface eth0 {
#          route-test {
#              count {
#                  failure 1
#                  success 1
#              }
#              type {
#                  script /config/scripts/network_test.sh
#              }
#          }
#      }
#      interface eth2 {
#          failover-only
#          route-test {
#              count {
#                  failure 1
#                  success 1
#              }
#              type {
#                  script /config/scripts/network_test.sh
#              }
#          }
#      }
#      sticky {
#          dest-addr enable
#          dest-port enable
#          source-addr enable
#      }
#      transition-script /config/scripts/wan-event-report.sh
#  }

PING_HOSTS='1.1.1.1 8.8.8.8'
MAX_AVERAGE_PING=50
MAX_PACKET_LOSS=0
PING_COUNT=5

GROUP=${1:-unknown}
IFACE=${2:-eth0}
STATE=${3:-DOWN}

# floating point greater than
fp_gt() {
    # NOTE: in bash 0 - true, 1 - false
    return $((1 - $(echo "$1 $2" | awk '{print ($1 > $2)}')))
}

# average of n floats (space separated)
fp_avg() {
    echo $(echo $@ | awk "{ for (i = 1; i <= NF; i++) s = s+\$i }; END { print s / NF }")
}

# logging function
log() {
    logger -t "network_test" -- "[$GROUP : $IFACE ($STATE)] $@"
    debug $@
}

debug() {
    echo "[$(date)] [$GROUP : $IFACE ($STATE)] $@" #>> /tmp/network_test.log
}

# examine ping
do_ping() {
    PACKET_LOSS_RE='[^0-9.]([0-9.]+)% packet loss'
    AVERAGE_PING_RE='[0-9.]+\/([0-9.]+)\/[0-9.]+\/[0-9.]'
    PINGOUT=$(/bin/ping -c $PING_COUNT -I $IFACE $1)

    if [[ $PINGOUT =~ $PACKET_LOSS_RE ]]; then
        PACKET_LOSS=${BASH_REMATCH[1]}
    fi

    if [[ $PINGOUT =~ $AVERAGE_PING_RE ]]; then
        AVERAGE_PING=${BASH_REMATCH[1]}
    fi

    echo "$(echo $PACKET_LOSS | awk '{print $1+0}') $(echo $AVERAGE_PING | awk '{print $1+0}')"
}

# dump traceroute
log_traceroute() {
    set -f # turns off globs
    log $(/usr/bin/traceroute -rnIi $IFACE -w 1 -q 1 1.1.1.1)
}

LOSSES=""
PINGS=""

for host in $(echo $PING_HOSTS); do
    PING_OUT=$(do_ping $host)
    debug "$host: $PING_OUT"
    LOSSES="$LOSSES $(echo $PING_OUT | awk '{print $1}')"
    PINGS="$PINGS $(echo $PING_OUT | awk '{print $2}')"
done

AVG_LOSS=$(fp_avg "$LOSSES")
AVG_PING=$(fp_avg "$PINGS")

if fp_gt $AVG_LOSS $MAX_PACKET_LOSS; then
    log "Too much packetloss: $AVG_LOSS% > $MAX_PACKET_LOSS%"
    log_traceroute &
    exit 1
fi

if fp_gt $AVG_PING $MAX_AVERAGE_PING; then
    log "Ping too high: ${AVG_PING}ms > ${MAX_AVERAGE_PING}ms"
    log_traceroute &
    exit 1
fi

exit 0