BASH benchmarking and latency monitoring

benchmark_function_v1.bash

#!/bin/bash

# (C)2020 Henri Shustak
# Relased Under MIT LICENCE
# https://mit-license.org/

# benchmark bash function
# requires gdate to be installed (part of coreutils)

# This system is very basic and has some serious limitiations.
# (1) - only supports milisecond benchmarking
# (2) - you will likely want to add some additional conditions and branches depending on the kind of benchmark you are running
#       for example you may want / require multiple benchmark functions to drill into a particular problem.

# benchmark data is in ms - if you want to add thousands seperator to make it more human readable pull request welcome.

#create log file for output
benchmark_log=`mktemp /tmp/benchmark.XXXXX` &&  mv -i "${benchmark_log}" "${temp}.log" && benchmark_log=`echo "${benchmark_log}.log"`
echo "writing to temporary log file : ${benchmark_log}"
echo

# add these lines and function to your script or source it in from your project
start_benchmark=""
end_benchmark=""
function benchmark() {
        if [ "${start_benchmark}" == "" ] ; then
                start_benchmark="`gdate +%s%N`"
        else
                end_benchmark="`gdate +%s%N`"
                elapsed_ms="`echo $(( end_benchmark - start_benchmark )) | sed 's/......$//'`"
                echo "${1}  ==>  ${elapsed_ms}ms" >> "${benchmark_log}"
                start_benchmark="$end_benchmark"
        fi
}


# usage example
benchmark
sleep 1
benchmark hello
benchmark my_other_benchmark

latency_monitor.bash

#!/bin/bash

# (C)2020 Henri Shustak
# Relased Under MIT LICENCE
# https://mit-license.org/

# This is a simple script which can be started to monitor the output from the benchmark process and then 
# take action / send alerts etc when latency exceeds a certin threshold for a certin operation
# for any kind of prodcution or longer term monitoring (especically on systems with limited storage
# you will probably want to setup some sort of log-rotation.

###
### Setup 
### (1) Name the log file as needed for your purposes.
### (2) remove any lines you would like to ignore with the grep -v call - add more if needed
### (3) configure actions and logic when latency is exceeded

tail -f /tmp/benchmark.XXXXX.log | grep  --line-buffered -v "line_delay" |
while read line ; do

	duration_extracted=`echo "${line}" |  gawk -F "  ==>  " '{print $2 ; system("") }' | sed 's/ms//'`

	# if latancey is more than 5 seconds for a any operation which is being monitored send notification
	if [ ${duration_extracted} -ge 5000 ] ; then 

		echo "latency theshold exceeded - sending push notification and restarting the backend system etc..." # reported via stdout
                # add your command and logic to be used / executred when the latency is exceeded 

	fi


done