Kill the process to prevent 100% disk usage

disk-guardian

#!/bin/bash
# Kill the top disk-writing process when a filesystem exceeds a usage threshold.
# Usage: disk-guardian [threshold%] [mountpoint]

# /tmp/disk-guardian 

THRESHOLD="${1:-95}"
MOUNTPOINT="${2:-/}"
LOG="/var/log/disk-guardian.log"

log() { echo "$(date '+%F %T') $*" | tee -a "$LOG" | logger -t disk-guardian; }

# Check current usage
usage=$(df --output=pcent "$MOUNTPOINT" | tail -1 | tr -d ' %')
(( usage < THRESHOLD )) && exit 0

log "Disk at ${usage}% on $MOUNTPOINT (threshold ${THRESHOLD}%) — finding top writer"

# Find the process with the highest cumulative write_bytes in /proc/<pid>/io.
# Skips kernel threads (no cmdline) and PIDs we cannot read.
top_pid=""
top_bytes=0

for io_file in /proc/[0-9]*/io; do
    pid="${io_file%/io}"; pid="${pid##*/proc/}"
    [[ -r "$io_file" ]] || continue
    [[ -r "/proc/$pid/cmdline" && -s "/proc/$pid/cmdline" ]] || continue  # skip kernel threads
    bytes=$(awk '/^write_bytes:/{print $2; exit}' "$io_file" 2>/dev/null) || continue
    if (( bytes > top_bytes )); then
        top_bytes=$bytes
        top_pid=$pid
    fi
done

if [[ -z "$top_pid" ]]; then
    log "ERROR: could not identify top writer — no action taken"
    exit 1
fi

comm=$(ps -p "$top_pid" -o comm=        2>/dev/null || echo "?")
user=$(ps -p "$top_pid" -o user=        2>/dev/null || echo "?")
cmd=$(cat "/proc/$top_pid/cmdline" 2>/dev/null | tr '\0' ' ' | cut -c1-120)
size_hr=$(numfmt --to=iec "$top_bytes" 2>/dev/null || echo "${top_bytes}B")

log "SIGTERM → PID $top_pid ($comm, user=$user, wrote=$size_hr): $cmd"
kill -TERM "$top_pid" 2>/dev/null || { log "PID $top_pid already gone"; exit 0; }

# Give it 10 seconds to clean up, then force-kill
for (( i=0; i<10; i++ )); do
    sleep 1
    kill -0 "$top_pid" 2>/dev/null || { log "PID $top_pid exited cleanly"; exit 0; }
done

log "SIGKILL → PID $top_pid ($comm) did not exit after 10s"
kill -KILL "$top_pid" 2>/dev/null || true

Another two files are needed, disk-guardian.service and disk-guardian.timer.

# /tmp/disk-guardian.service
[Unit]
Description=Kill top disk writer when filesystem exceeds threshold

[Service]
Type=oneshot
ExecStart=/usr/local/bin/disk-guardian 95 /

# /tmp/disk-guardian.timer
[Unit]
Description=Run disk-guardian every 300 seconds

[Timer]
OnBootSec=600
OnUnitActiveSec=300s
AccuracySec=5s

[Install]
WantedBy=timers.target

An here is how to install it:

sudo install -m 755 /tmp/disk-guardian /usr/local/bin/disk-guardian                                                                                                                                                                
sudo install -m 644 /tmp/disk-guardian.service /etc/systemd/system/disk-guardian.service                                                                                                                                           
sudo install -m 644 /tmp/disk-guardian.timer /etc/systemd/system/disk-guardian.timer                                                                                                                                               
sudo systemctl daemon-reload                                                                                                                                                                                                       
sudo systemctl enable --now disk-guardian.timer                                                                                                                                                                                    
                                                                                                                                                                                                                                       
#  Then verify it's running:                                                                                                                                                                                                            
                                                                                                                                                                                                                                       
sudo systemctl status disk-guardian.timer                                                                                                                                                                                          
sudo systemctl list-timers disk-guardian.timer