UniFi OS Server: diagnose & fix updates failing due to internal cert missing a 127.0.0.1 SAN (custom cert clobbered unifi-core.crt)

uos-supervisor-cert-fix.sh

#!/usr/bin/env bash
#
# uos-supervisor-cert-fix.sh
#
# UniFi OS Server: diagnose and fix "update keeps failing" caused by a custom
# TLS certificate having been written over the INTERNAL cert (unifi-core.crt).
#
# The supervisor / update control channel connects to the core over loopback
# (https://127.0.0.1:<supervisor-port>) with strict TLS hostname verification.
# The internal cert must therefore carry a 127.0.0.1 SAN. When a user installs a
# public/custom cert into the system slot, that cert (SAN = public hostname only)
# replaces unifi-core.crt, loopback verification fails, and NO updates apply.
#
# The fix restores the factory internal cert from its *.bak sidecar. It does NOT
# touch the UI certificate (settings.yaml activeCertId / http/local-certs.conf),
# so a trusted public cert on the web UI is preserved.
#
# Usage:
#   sudo ./uos-supervisor-cert-fix.sh diagnose     # read-only, default
#   sudo ./uos-supervisor-cert-fix.sh fix [-y]     # restores cert, restarts core
#   ./uos-supervisor-cert-fix.sh selftest          # validate detector, no root
#
# Exit codes (diagnose): 0 healthy · 10 affected · 3 undetermined · 2 error
#
set -euo pipefail

MODE="${1:-diagnose}"
ASSUME_YES=0; [ "${2:-}" = "-y" ] || [ "${2:-}" = "--yes" ] && ASSUME_YES=1

SVCLOG="/var/lib/uosserver/logs/uosserver-service.log"
LOOPBACK_SAN='IP Address:127\.0\.0\.1'

log()  { printf '%s\n' "$*"; }
err()  { printf 'ERROR: %s\n' "$*" >&2; }
die()  { err "$*"; exit 2; }

need() { command -v "$1" >/dev/null 2>&1 || die "missing required tool: $1"; }

# --- cert helpers (RSA + EC safe) -------------------------------------------

# true if cert file's SAN contains an IP:127.0.0.1 entry
san_has_loopback() {
  openssl x509 -in "$1" -noout -ext subjectAltName 2>/dev/null \
    | grep -qE "$LOOPBACK_SAN"
}

# true if cert ($1) and key ($2) are a matching pair (public keys equal)
pair_ok() {
  local cpub kpub
  cpub=$(openssl x509 -in "$1" -noout -pubkey 2>/dev/null) || return 1
  kpub=$(openssl pkey -in "$2" -pubout      2>/dev/null) || return 1
  [ -n "$cpub" ] && [ "$cpub" = "$kpub" ]
}

cert_subject() { openssl x509 -in "$1" -noout -subject 2>/dev/null | sed 's/^subject=//'; }

# SAN of the cert actually served on a TLS port (what the client really sees)
served_san_has_loopback() {
  echo | openssl s_client -connect "127.0.0.1:$1" 2>/dev/null \
    | openssl x509 -noout -ext subjectAltName 2>/dev/null \
    | grep -qE "$LOOPBACK_SAN"
}

# --- discovery (no hardcoded hostnames, UUIDs, or volume paths) --------------

find_config_dir() {
  local uhome cand
  uhome=$(getent passwd uosserver 2>/dev/null | cut -d: -f6 || true)
  # 1) rootless podman default storage under the uosserver user's home
  if [ -n "$uhome" ]; then
    cand="$uhome/.local/share/containers/storage/volumes/uosserver_data/_data/unifi-core/config"
    [ -f "$cand/unifi-core.crt" ] && { printf '%s' "$cand"; return 0; }
  fi
  # 2) ask podman for the volume mountpoint
  cand=$(sudo -u uosserver podman volume inspect uosserver_data \
           --format '{{.Mountpoint}}' 2>/dev/null || true)
  if [ -n "$cand" ] && [ -f "$cand/unifi-core/config/unifi-core.crt" ]; then
    printf '%s' "$cand/unifi-core/config"; return 0
  fi
  # 3) last resort: search the filesystem
  cand=$(find / -xdev -type f -name unifi-core.crt -path '*unifi-core/config/*' \
           2>/dev/null | head -1 || true)
  [ -n "$cand" ] && { dirname "$cand"; return 0; }
  return 1
}

# supervisor TLS port from its generated vhost (fallback: 11084)
supervisor_port() {
  local conf="$1/http/site-supervisor.conf" p
  p=$(grep -m1 -oE 'listen[[:space:]]+[0-9]+' "$conf" 2>/dev/null | grep -oE '[0-9]+' || true)
  printf '%s' "${p:-11084}"
}

# --- selftest: prove the loopback-SAN detector works on this openssl ---------

run_selftest() {
  need openssl
  SELFTEST_TMP=$(mktemp -d)
  trap 'rm -rf "${SELFTEST_TMP:-}"' EXIT
  local d="$SELFTEST_TMP"
  # GOOD: cert WITH loopback SAN. BAD: cert WITHOUT it.
  openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj '/CN=unifi.local' \
    -addext 'subjectAltName=DNS:unifi.local,DNS:localhost,IP:127.0.0.1' \
    -keyout "$d/g.key" -out "$d/g.crt" >/dev/null 2>&1
  openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj '/CN=example.com' \
    -addext 'subjectAltName=DNS:example.com' \
    -keyout "$d/b.key" -out "$d/b.crt" >/dev/null 2>&1
  san_has_loopback "$d/g.crt" || die "selftest FAIL: loopback cert not detected"
  ! san_has_loopback "$d/b.crt" || die "selftest FAIL: non-loopback cert misflagged"
  pair_ok "$d/g.crt" "$d/g.key" || die "selftest FAIL: matching pair rejected"
  ! pair_ok "$d/g.crt" "$d/b.key" || die "selftest FAIL: mismatched pair accepted"
  log "selftest OK (detector + pair check behave correctly)"
}

# --- diagnosis --------------------------------------------------------------
# Sets globals: CFG INT_CRT INT_KEY BAK_CRT BAK_KEY PORT AFFECTED
diagnose_core() {
  need openssl; need uosserver
  CFG=$(find_config_dir) || die "could not locate unifi-core config dir; is UniFi OS Server installed?"
  INT_CRT="$CFG/unifi-core.crt"; INT_KEY="$CFG/unifi-core.key"
  BAK_CRT="$CFG/unifi-core.crt.bak"; BAK_KEY="$CFG/unifi-core.key.bak"
  PORT=$(supervisor_port "$CFG")
  [ -f "$INT_CRT" ] || die "internal cert not found: $INT_CRT"

  log "config dir       : $CFG"
  log "internal cert    : $INT_CRT"
  log "  subject        : $(cert_subject "$INT_CRT")"
  log "supervisor port  : $PORT"

  local file_bad=0 served_bad=0
  san_has_loopback "$INT_CRT" || file_bad=1
  if served_san_has_loopback "$PORT"; then served_bad=0; else served_bad=1; fi

  log "internal cert has 127.0.0.1 SAN : $([ $file_bad -eq 0 ] && echo yes || echo NO)"
  log "served cert (:$PORT) has it     : $([ $served_bad -eq 0 ] && echo yes || echo NO)"

  # corroborate from the log if present
  if [ -f "$SVCLOG" ]; then
    local last
    last=$(grep -iE 'Supervisor (connected|error connecting)' "$SVCLOG" 2>/dev/null | tail -1 || true)
    [ -n "$last" ] && log "last supervisor event : ${last#* }"
  fi

  AFFECTED=0
  if [ $file_bad -eq 1 ] || [ $served_bad -eq 1 ]; then AFFECTED=1; fi
}

do_diagnose() {
  diagnose_core
  echo
  if [ "$AFFECTED" -eq 1 ]; then
    log "VERDICT: AFFECTED — internal cert lacks a 127.0.0.1 SAN; the supervisor"
    log "         update channel cannot complete its loopback TLS handshake."
    if [ -f "$BAK_CRT" ] && [ -f "$BAK_KEY" ] && san_has_loopback "$BAK_CRT" && pair_ok "$BAK_CRT" "$BAK_KEY"; then
      log "         Factory backup present and valid -> run: sudo $0 fix"
    else
      log "         No usable factory backup (*.bak) found -> see notes below."
    fi
    exit 10
  fi
  log "VERDICT: healthy — internal/supervisor cert carries the loopback SAN."
  exit 0
}

# --- fix --------------------------------------------------------------------

do_fix() {
  [ "$(id -u)" -eq 0 ] || die "fix must run as root"
  diagnose_core
  echo
  if [ "$AFFECTED" -ne 1 ]; then
    log "Nothing to fix — internal cert already valid for loopback."
    exit 0
  fi

  # Require a valid factory backup; never fabricate a cert.
  [ -f "$BAK_CRT" ] && [ -f "$BAK_KEY" ] || \
    die "no factory backup ($BAK_CRT / .key). Reinstall UniFi OS Server or regenerate the internal cert with a 127.0.0.1 SAN manually."
  san_has_loopback "$BAK_CRT" || die "backup cert lacks 127.0.0.1 SAN; refusing to use it."
  pair_ok "$BAK_CRT" "$BAK_KEY" || die "backup cert/key are not a matching pair; refusing to use it."

  log "Plan: restore factory internal cert, then restart the UOS container."
  log "  from : $BAK_CRT (subject $(cert_subject "$BAK_CRT"))"
  log "  onto : $INT_CRT"
  log "  UI cert (activeCertId / local-certs.conf) is left untouched."
  if [ "$ASSUME_YES" -ne 1 ]; then
    read -r -p "Proceed? This briefly restarts UniFi OS Server. [y/N] " a
    case "$a" in y|Y) ;; *) log "aborted."; exit 0;; esac
  fi

  # capture ownership/mode of the current internal cert to reapply
  local own mode ts
  own=$(stat -c '%U:%G' "$INT_CRT"); mode=$(stat -c '%a' "$INT_CRT")
  ts=$(date -u +%Y%m%dT%H%M%SZ)

  log "Backing up current internal cert -> *.custom.$ts"
  cp -a "$INT_CRT" "$INT_CRT.custom.$ts"
  cp -a "$INT_KEY" "$INT_KEY.custom.$ts"

  log "Restoring factory internal cert..."
  cp "$BAK_CRT" "$INT_CRT"
  cp "$BAK_KEY" "$INT_KEY"
  chown "$own" "$INT_CRT" "$INT_KEY"
  chmod "$mode" "$INT_CRT" "$INT_KEY"
  san_has_loopback "$INT_CRT" || die "post-copy check failed: restored cert still lacks loopback SAN"

  # mark log position so we only read post-restart events
  local before=0; [ -f "$SVCLOG" ] && before=$(wc -l < "$SVCLOG")

  log "Restarting UniFi OS Server (uosserver stop && start)..."
  uosserver stop  >/dev/null 2>&1 || true
  uosserver start >/dev/null 2>&1 || die "uosserver start failed; internal cert restored, restart manually"

  log "Waiting for the supervisor to reconnect (up to ~180s)..."
  local ok=0 deadline=$((SECONDS+180))
  while [ $SECONDS -lt $deadline ]; do
    if [ -f "$SVCLOG" ]; then
      local new
      new=$(tail -n +"$((before+1))" "$SVCLOG" 2>/dev/null | grep -iE 'Supervisor (connected|error connecting)' | tail -1 || true)
      case "$new" in
        *"Supervisor connected"*)          ok=1; break;;
      esac
    fi
    served_san_has_loopback "$PORT" && { ok=2; }   # cert is right; keep waiting briefly for connect
    sleep 6
  done

  echo
  if [ "$ok" -eq 1 ]; then
    log "SUCCESS: supervisor reconnected. Updates should now work."
  elif [ "$ok" -eq 2 ] || served_san_has_loopback "$PORT"; then
    log "Cert restored and the supervisor endpoint now serves a loopback-valid cert."
    log "The host retries the channel every ~5 min; it should connect shortly."
  else
    log "Cert restored, but reconnection not yet confirmed in the log."
    log "Check: grep 'Supervisor' $SVCLOG | tail"
  fi

  echo
  log "Notes:"
  log " - Your web-UI certificate was NOT changed (external trust preserved)."
  log " - Do not install a custom cert into the SYSTEM/internal slot again; keep"
  log "   custom certs on the UI cert only, or this breaks the same way."
  log " - Rollback if ever needed: cp \"$INT_CRT.custom.$ts\" \"$INT_CRT\" (and .key), then uosserver stop && start."
}

case "$MODE" in
  diagnose|--diagnose) do_diagnose ;;
  fix|--fix)           do_fix ;;
  selftest|--selftest) run_selftest ;;
  -h|--help|help)      sed -n '2,30p' "$0" ;;
  *) die "unknown mode '$MODE' (use: diagnose | fix [-y] | selftest)";;
esac