On remote Linux, track system load averaging and implement job with e-mail notification

ninja-job.bash

#!/usr/bin/env bash

# NOTE ASSUMPTION OF LINUX
# NOTE ENVIRONMENT VARIABLES THAT MUST BE SET IN ADVANCE (OR REPLACED):
#   export EMAIL='[email protected]'
#   export JOB='python path/to/src.py'
#   export JOBNAME='python post processing'
# RUN THIS SCRIPT WITH NOHUP OR SCREEN OR TMUX

loadavg() {
  cat /proc/loadavg | awk '{print (($1 + $2 + $3)<=3?1:0)}'
}

count=0
kount=0

# Increments $count when loadavg is below threshold, resets $count when
# loadavg is above threshold. If $count is below threshold for two
# consecutive periods, while loop ends and job begins. $kount tracks
# consecutive periods.

while true; do
  res=$(loadavg)
  [[ $res -eq  1 ]] && let count++ || count=0
  sleep 600
  [[ $count -gt 0 ]] && let kount++ || kount=0
  [[ $kount -eq 2 ]] && break || :
done

mail -s 'Starting job' $EMAIL << __EOF 
  Starting $JOBNAME on $HOSTNAME.
__EOF

$JOB

mail -s 'Finished job' $EMAIL << __EOF 
  Finished $JOBNAME on $HOSTNAME.
__EOF