llama-gemma-abliterated.sh

#!/bin/bash
~/killer.sh llama-server
LLAMA_DIR=~/Documents/GitHub/llama
mkdir -p "$LLAMA_DIR"
cd "$LLAMA_DIR"
cp ~/llama-cpp-updater.sh .


# sudo apt-get install gcc-13 g++-13
export CC=/usr/bin/gcc-13
export CXX=/usr/bin/g++-13
# Then re-run your llama-deepseek-v4.sh and choose option 1 again

# ── Choose llama.cpp source (upstream vs fork) ──────────────────────────────
echo ""
echo "  Choose llama.cpp source:"
echo "  1) Upstream ggml-org/llama.cpp (default)"
echo "  2) Fork am17an/llama.cpp (branch: mtp-clean)"
echo -n "  Choose [1/2] or Enter: "
read -r repo_choice

UPDATER_REPO="ggml-org/llama.cpp"
UPDATER_BRANCH=""
case "${repo_choice:-1}" in
  2)
    UPDATER_REPO="am17an/llama.cpp"
    UPDATER_BRANCH="mtp-clean"
    ;;
esac

UPDATER_ARGS=(--repo "$UPDATER_REPO")
[ -n "$UPDATER_BRANCH" ] && UPDATER_ARGS+=(--branch "$UPDATER_BRANCH")

# ── Find best CUDA build ─────────────────────────────────────────────────────
CUDA_BIN=""
CUDA_LABEL=""
for d in $(ls -d "$LLAMA_DIR"/b*-cuda "$LLAMA_DIR"/llama-b*-cuda 2>/dev/null | sort -rV); do
    [ -x "$d/bin/llama-server" ] && CUDA_BIN="$d/bin/llama-server" && CUDA_LABEL="$(basename $d)" && break
    [ -x "$d/llama-server" ]     && CUDA_BIN="$d/llama-server"     && CUDA_LABEL="$(basename $d)" && break
done

# ── If CUDA exists, show 3-option prompt ────────────────────────────────────
if [ -n "$CUDA_BIN" ]; then
  LATEST_VERSION=$(curl -sf "https://api.github.com/repos/$UPDATER_REPO/releases/latest" \
      | grep -o '"tag_name": *"[^"]*"' | head -n1 | cut -d'"' -f4 2>/dev/null || echo "unknown")

    echo ""
    echo "  1) Use existing CUDA build  [$CUDA_LABEL]  (default)"
    echo "  2) Build updated CUDA       [latest: $LATEST_VERSION, ~10 min]"
    echo "  3) Download Vulkan binary   [pre-built, ~2 min]"
    echo -n "  Choose [1/2/3] or Enter: "
    read -r choice

    case "${choice:-1}" in
        2) ./llama-cpp-updater.sh --cuda --yes "${UPDATER_ARGS[@]}"
           CHOSEN_BIN="$LLAMA_DIR/llama.cpp/bin/llama-server"
           [ ! -x "$CHOSEN_BIN" ] && CHOSEN_BIN="$LLAMA_DIR/llama.cpp/llama-server" ;;
        3) ./llama-cpp-updater.sh --vulkan --yes "${UPDATER_ARGS[@]}"
           CHOSEN_BIN="$LLAMA_DIR/llama.cpp/llama-server"
           [ ! -x "$CHOSEN_BIN" ] && CHOSEN_BIN="$LLAMA_DIR/llama.cpp/bin/llama-server" ;;
        *) CHOSEN_BIN="$CUDA_BIN" ;;
    esac

else
    # ── No CUDA build — offer download Vulkan or build CUDA ─────────────────
  LATEST_VERSION=$(curl -sf "https://api.github.com/repos/$UPDATER_REPO/releases/latest" \
      | grep -o '"tag_name": *"[^"]*"' | head -n1 | cut -d'"' -f4 2>/dev/null || echo "unknown")

    echo ""
    echo "  No CUDA build found. (GitHub latest: $LATEST_VERSION)"
    echo "  1) BUILD CUDA  (~10 min, best performance)"
    echo "  2) Download Vulkan pre-built (~2 min)"
    echo -n "  Choose [1/2] (default: 1): "
    read -r choice
    case "${choice:-1}" in
      2) ./llama-cpp-updater.sh --vulkan --yes "${UPDATER_ARGS[@]}"
           CHOSEN_BIN="$LLAMA_DIR/llama.cpp/llama-server"
           [ ! -x "$CHOSEN_BIN" ] && CHOSEN_BIN="$LLAMA_DIR/llama.cpp/bin/llama-server" ;;
      *) ./llama-cpp-updater.sh --cuda --yes "${UPDATER_ARGS[@]}"
           CHOSEN_BIN="$LLAMA_DIR/llama.cpp/bin/llama-server"
           [ ! -x "$CHOSEN_BIN" ] && CHOSEN_BIN="$LLAMA_DIR/llama.cpp/llama-server" ;;
    esac
fi

if [ ! -x "$CHOSEN_BIN" ]; then
    echo "ERROR: llama-server not found at: $CHOSEN_BIN"
    exit 1
fi


if [ ! -x "$CHOSEN_BIN" ]; then
    echo "ERROR: llama-server not found at: $CHOSEN_BIN"
    exit 1
fi

# If binary is in bin/, prepend the sibling lib/ so shared libs are found
BIN_PARENT=$(dirname "$CHOSEN_BIN")
if [ "$(basename "$BIN_PARENT")" = "bin" ]; then
    LIB_DIR="$(dirname "$BIN_PARENT")/lib"
    [ -d "$LIB_DIR" ] && export LD_LIBRARY_PATH="$LIB_DIR${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
fi

echo "  Starting: $CHOSEN_BIN"

# ── Install as systemd service ───────────────────────────────────────────────
SERVICE_NAME="llama-server"
SERVICE_FILE="/etc/systemd/system/${SERVICE_NAME}.service"

echo ""
echo -n "  Install/Update as systemd service? [y/N] "
read -r install_svc

if [[ "$install_svc" =~ ^[Yy]$ ]]; then
    echo "  Installing optimized systemd service for RTX 5090..."

    sudo tee "$SERVICE_FILE" >/dev/null <<EOF
[Unit]
Description=llama-server (Gemma-4 12B Abliterated on RTX 5090)
After=network.target

[Service]
Type=simple
User=$USER
Environment=LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}
ExecStart=$CHOSEN_BIN \\
  -hf mradermacher/gemma-4-12B-it-abliterated-uncensored-GGUF:Q4_K_M \\
  --host 0.0.0.0 \\
  --port 8090 \\
  --ctx-size 131072 \\
  --n-gpu-layers 99 \\
  --flash-attn on \\
  --cache-type-k q4_0 \\
  --cache-type-v q4_0 \\
  --parallel 1 \\
  --cont-batching \\
  --jinja \\
  --alias "gemma-4-12b-ablit" \\
  --api-key 1234567890 \\
  --no-webui \\
  -ub 512 \\
  --no-mmap
Restart=on-failure
RestartSec=10
LimitNOFILE=65535

[Install]
WantedBy=multi-user.target
EOF

    sudo systemctl daemon-reload
    sudo systemctl enable "$SERVICE_NAME"
    sudo systemctl restart "$SERVICE_NAME"
    echo "  Service installed and restarted with 128k context optimization."
    echo "  Check status: systemctl status llama-server"
    echo "  Logs:         journalctl -u llama-server -f"
    exit 0
fi

# ── Direct run (for testing) ─────────────────────────────────────────────────
echo "  Running directly with optimized settings (128k context)..."

exec "$CHOSEN_BIN" \
  -hf mradermacher/gemma-4-12B-it-abliterated-uncensored-GGUF:Q4_K_M \
  --host 0.0.0.0 \
  --port 8090 \
  --ctx-size 131072 \
  --n-gpu-layers 99 \
  --flash-attn on \
  --cache-type-k q4_0 \
  --cache-type-v q4_0 \
  --parallel 1 \
  --cont-batching \
  --jinja \
  --alias "gemma-4-12b-ablit" \
  --api-key 1234567890 \
  --no-webui \
  -ub 512 \
  --no-mmap


curl http://127.0.0.1:8090/v1/chat/completions \
  -H "Authorization: Bearer 1234567890" \
  -H "Content-Type: application/json" \
  -d '{
    "model": "gemma-4-12b-ablit",
    "messages": [{"role": "user", "content": "Say a random number between 1 and 1000"}],
    "stream": false
  }'