Detect Processes in "D" State & Reboot via SysRq After Sending Matrix Alert

d_stuck.sh

#!/bin/bash

PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin

function matrixMessage {
	local INPUTMSG=$1

	local MATRIX_SERVERURL="matrix.server.com" #dont add http(s)
	local MATRIX_ROOMID="!xXxXxXxXxXxX" #Format: !XxXxXxXxXxXxXx
	local MATRIX_TXNID="$(date +%s)"
	local MATRIX_APIKEY="xXxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXxX..."

	local MESSAGE=".:: $(readlink -f $0) on $(hostname -s) ::."$'\n\n'"${INPUTMSG}"
	local MESSAGE=$(python3 -c 'import json,sys; print(json.dumps(sys.stdin.read()))' <<< $MESSAGE) # use python

	curl --max-time 10 -s -X PUT --header 'Content-Type: application/json' --header 'Accept: application/json' --data-raw "{\"msgtype\":\"m.text\", \"body\":${MESSAGE}}" "https://${MATRIX_SERVERURL}/_matrix/client/r0/rooms/${MATRIX_ROOMID}:${MATRIX_SERVERURL}/send/m.room.message/${MATRIX_TXNID}?access_token=${MATRIX_APIKEY}"
}

# get processes stuck in D state that were started more than 360 minutes (6 hours) ago
STUCK_PROCESSES=$(ps -eo lstart,state,cmd --no-headers | awk '$6 == "D"' | while read i; do STARTDATE=$(date -d "$(awk '{print $1,$2,$3,$4,$5}' <<< $i)" +"%s"); NOWDATE=$(date +"%s"); MINSTUCK=$((($NOWDATE-$STARTDATE)/60)); if [ $MINSTUCK -ge 360 ]; then echo "$i -- Stuck for $MINSTUCK minutes"; fi done)

if [ $(wc -l <<< $STUCK_PROCESSES) -ge 3 ]; then
  MSG=$'*********\n'"SysRq rebooting because more than three processes have been stuck in the \"D\" state for over 6 hours."$'\n\n'
  MSG="${MSG}PROCESSES:"$'\n'"${STUCK_PROCESSES}"$'\n*********'
  matrixMessage "$MSG"

  logger -s -p user.emerg '!!! WARNING !!! SYSTEM WILL REBOOT IN LESS THAN ONE MINUTE BECAUSE OF STUCK PROCESSES !!! WARNING !!!'
  wall "!!! WARNING !!! SYSTEM WILL REBOOT IN LESS THAN ONE MINUTE BECAUSE OF STUCK PROCESSES !!! WARNING !!!"

  echo 1 >/proc/sys/kernel/sysrq
  echo s >/proc/sysrq-trigger; sleep 10
  echo u >/proc/sysrq-trigger; sleep 5
  echo b >/proc/sysrq-trigger
fi