Plex Docker watchdog via crontab -e with Telegram alerts & DB‑busy protection "Sqlite3: Sleeping for 200ms to retry busy DB."

plex_health_watchdog.sh

#!/usr/bin/env bash
#
# plex_health_watchdog.sh — Plex Docker watchdog with Telegram alerts & DB‑busy protection
#
# WHY IT’S USEFUL
# ----------------
# Keeps your Plex Media Server Docker container healthy by:
#  1. Monitoring HTTP responsiveness (restarts after 2 consecutive curl failures).
#  2. Detecting “busy DB” storms (restarts if > MAX_DB_BUSY in LOG_TAIL lines).
#  3. Detecting a persistent single “busy DB” lock (restarts if it lasts ≥ THRESHOLD seconds).
#  4. (Optional) Sending Telegram alerts via Apprise on any restart — and even an “all clear” message.
#
# These issues are rare but can happen under load or transient network hiccups. This script
# is your “last‑resort” auto‑fix. E.g.:  
#   “It happens extremely rarely on my server, but if it does, the script will resolve it by restarting the container,
#    which should get Plex going again.”
#
# DEPENDENCIES
# -------------
# • docker CLI (no sudo) — see https://docs.docker.com/engine/install/linux-postinstall/
# • curl
# • apprise CLI (for Telegram) — install via:
#     pip3 install --user apprise
#     # or on Debian/Ubuntu:
#     sudo apt-get install apprise
#
# INSTALLATION
# -------------
# 1. Save this file as ~/plex_health_watchdog.sh
# 2. chmod +x ~/plex_health_watchdog.sh
# 3. Add to your crontab via "crontab -e" (runs every 3 minutes in this example):
#      */3 * * * * "/home/you/plex_health_watchdog.sh" >/dev/null 2>&1
#
### ─── Configuration ──────────────────────────────────────────────────────────
# Plex HTTP/API
PLEX_TOKEN='YOUR_PLEX_TOKEN_HERE'        # fill in your Plex token
PLEX_IP='PLEX_HOST_IP_OR_HOSTNAME'      # e.g. 192.168.1.100 or plex.example.com
PLEX_PORT='32400'                         # Plex default port; change if customized

# Docker container name, change if necessary
CONTAINER_NAME='plex'

# ─ Burst‑mode “busy DB” protection ────────────────────────────────────────────
LOG_TAIL=10                             # how many recent lines to inspect
MAX_DB_BUSY=4                           # max allowed “busy DB” msgs in that window
BUSY_MSG='Sqlite3: Sleeping for 200ms to retry busy DB.'

# ─ Persistent “busy DB” protection ───────────────────────────────────────────
THRESHOLD=600                           # seconds before a single-line event triggers
                                        # (600 = 10 minutes)

# ─ HTTP hiccup protection ────────────────────────────────────────────────────
# first curl failure creates marker; second consecutive failure triggers restart
# marker file lives alongside this script:
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
CURL_MARKER="$SCRIPT_DIR/plex_curl_fail.marker"
BUSY_MARKER="$SCRIPT_DIR/plex_busy.marker"

# ─ Optional Telegram via Apprise ──────────────────────────────────────────────
# Leave either blank to disable alerts completely
TELEGRAM_BOT_TOKEN=''                   # e.g. 123456789:ABCDefGhiJkLmNoPQRsTUvWxYz
TELEGRAM_CHAT_ID=''                     # e.g. 987654321
SEND_OKAY_ALERT=false                   # set to true to get an “all clear” message
OKAY_MSG='Plex is operating normally.'  # custom text for OK alert

# Apprise must be in your PATH
function send_alert() {
  local msg="$1"
  if [[ -n "$TELEGRAM_BOT_TOKEN" && -n "$TELEGRAM_CHAT_ID" ]]; then
    # use Apprise’s tgram:// scheme for Telegram
    apprise -t "Plex Watchdog" -b "$msg" \
      "tgram://${TELEGRAM_BOT_TOKEN}/${TELEGRAM_CHAT_ID}" \
      || echo "$(date): [WARN] failed to send Telegram alert" >&2
  fi
}
### ────────────────────────────────────────────────────────────────────────────

# 0) Only proceed if container is running
if ! docker inspect --format='{{.State.Running}}' "$CONTAINER_NAME" &>/dev/null; then
  exit 0
fi

# 1) HTTP health check (2‑fail restart)
if ! curl -sSf "http://${PLEX_IP}:${PLEX_PORT}/?X-Plex-Token=${PLEX_TOKEN}" -o /dev/null; then
  if [[ ! -f "$CURL_MARKER" ]]; then
    # first failure: touch marker
    touch "$CURL_MARKER"
  else
    # second consecutive failure → restart + alert
    echo "$(date): [ERROR] HTTP failed twice → restarting $CONTAINER_NAME" >&2
    docker restart "$CONTAINER_NAME"
    rm -f "$CURL_MARKER" "$BUSY_MARKER"
    send_alert "HTTP health check failed twice; Plex container restarted."
    exit 0
  fi
else
  # success → clear any previous HTTP-fail marker
  rm -f "$CURL_MARKER"
fi

# 2) Burst‑mode busy‑DB check
busy_count=$(docker logs --tail "$LOG_TAIL" "$CONTAINER_NAME" 2>/dev/null \
             | grep -F "$BUSY_MSG" | wc -l)
if (( busy_count > MAX_DB_BUSY )); then
  echo "$(date): [ERROR] $busy_count busy‑DB msgs (> $MAX_DB_BUSY) → restarting $CONTAINER_NAME" >&2
  docker restart "$CONTAINER_NAME"
  rm -f "$BUSY_MARKER" "$CURL_MARKER"
  send_alert "Detected $busy_count SQLite busy‑DB messages; Plex container restarted."
  exit 0
fi

# 3) Persistent single busy‑DB check
if docker logs --tail "$LOG_TAIL" "$CONTAINER_NAME" 2>/dev/null | grep -qF "$BUSY_MSG"; then
  if [[ ! -f "$BUSY_MARKER" ]]; then
    # first sighting: create marker (mtime marks start)
    touch "$BUSY_MARKER"
  else
    # check age of marker
    mtime=$(stat -c %Y "$BUSY_MARKER")
    now=$(date +%s)
    if (( now - mtime >= THRESHOLD )); then
      echo "$(date): [ERROR] busy‑DB persisted ≥$((THRESHOLD/60))min → restarting $CONTAINER_NAME" >&2
      docker restart "$CONTAINER_NAME"
      rm -f "$BUSY_MARKER" "$CURL_MARKER"
      send_alert "A single SQLite busy‑DB event persisted ≥$((THRESHOLD/60)) min; Plex restarted."
      exit 0
    fi
  fi
else
  # healthy run → reset busy marker
  rm -f "$BUSY_MARKER"
fi

# 4) Optional “all clear” alert
if [[ "$SEND_OKAY_ALERT" == "true" ]]; then
  send_alert "$OKAY_MSG"
fi

exit 0