run_model.sh

#!/usr/bin/env bash
#
# llama-serve.sh — wrap llama-server with sane defaults for M3 Ultra
# that approximate the LM Studio endpoint used by qwaude / SSH tunnel.
#
# Usage:
#   ./llama-serve.sh <path-to.gguf>
#
# Env overrides:
#   PORT      (default 1234)
#   HOST      (default 127.0.0.1)
#   CTX       (default 32768 — total; split across PARALLEL slots)
#   PARALLEL  (default 8)
#   NGL       (default 99 — full Metal offload)
#   FA        (default on  — Flash Attention; on|off|auto)
#   KV_Q      (default empty — set to "q8_0" or "q4_0" to quantize K/V cache)
#   ALIAS     (default derived from filename without .gguf extension)
#   EXTRA     (any additional llama-server flags, appended verbatim)

set -euo pipefail

# --- arg check ---------------------------------------------------------------
if [[ $# -lt 1 ]]; then
  echo "usage: $0 <path-to.gguf>" >&2
  exit 2
fi

MODEL="$1"
if [[ ! -f "$MODEL" ]]; then
  echo "error: model not found: $MODEL" >&2
  exit 1
fi

# --- llama-server presence ---------------------------------------------------
if ! command -v llama-server >/dev/null 2>&1; then
  echo "error: llama-server not on PATH (brew install llama.cpp)" >&2
  exit 1
fi

# --- defaults ----------------------------------------------------------------
PORT="${PORT:-1234}"
HOST="${HOST:-127.0.0.1}"
CTX="${CTX:-32768}"
PARALLEL="${PARALLEL:-8}"
NGL="${NGL:-99}"
FA="${FA:-on}"
KV_Q="${KV_Q:-}"
ALIAS="${ALIAS:-$(basename "$MODEL" .gguf)}"
EXTRA="${EXTRA:-}"

# --- KV cache quant (optional) -----------------------------------------------
KV_ARGS=()
if [[ -n "$KV_Q" ]]; then
  KV_ARGS=(-ctk "$KV_Q" -ctv "$KV_Q")
fi

# --- log what we're about to run --------------------------------------------
PER_SLOT=$(( CTX / PARALLEL ))
cat <<EOF >&2
llama-serve.sh
  model     : $MODEL
  alias     : $ALIAS
  endpoint  : http://$HOST:$PORT
  context   : $CTX total / $PARALLEL slots = $PER_SLOT per slot
  ngl       : $NGL
  fa        : $FA
  kv-quant  : ${KV_Q:-off (f16)}
  extra     : ${EXTRA:-<none>}

EOF

# --- run ---------------------------------------------------------------------
# shellcheck disable=SC2086  # EXTRA is intentionally word-split
exec llama-server \
  -m "$MODEL" \
  --host "$HOST" \
  --port "$PORT" \
  -c "$CTX" \
  --parallel "$PARALLEL" \
  -ngl "$NGL" \
  -fa "$FA" \
  --jinja \
  --alias "$ALIAS" \
  "${KV_ARGS[@]}" \
  $EXTRA