install_llm.sh

gistfile1.txt

#!/usr/bin/env bash
set -euo pipefail

INSTALL_DIR="$(cd "$(dirname "$0")" && pwd)"
LLAMA_DIR="$INSTALL_DIR/llama.cpp"
MODELS_DIR="$INSTALL_DIR/models"

echo "=== Installing llama.cpp ==="

# Dependencies
sudo apt-get update -qq
sudo apt-get install -y git build-essential cmake curl

# Clone or update llama.cpp
if [ -d "$LLAMA_DIR" ]; then
    echo "llama.cpp already cloned, pulling latest..."
    git -C "$LLAMA_DIR" pull
else
    git clone https://github.com/ggerganov/llama.cpp "$LLAMA_DIR"
fi

echo "=== Building llama.cpp (CPU, ARM optimized) ==="
cmake -S "$LLAMA_DIR" -B "$LLAMA_DIR/build" \
    -DCMAKE_BUILD_TYPE=Release \
    -DGGML_NATIVE=ON
cmake --build "$LLAMA_DIR/build" --config Release -j"$(nproc)"

echo "=== Downloading model weights ==="
mkdir -p "$MODELS_DIR"

MODEL_FILE="$MODELS_DIR/mistral-7b-instruct-v0.2.Q4_K_M.gguf"
MODEL_URL="https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.2-GGUF/resolve/main/mistral-7b-instruct-v0.2.Q4_K_M.gguf"

if [ -f "$MODEL_FILE" ]; then
    echo "Model already downloaded: $MODEL_FILE"
else
    echo "Downloading ~4.4GB model (Q4_K_M quantization)..."
    curl -L --progress-bar -o "$MODEL_FILE" "$MODEL_URL"
fi

echo "=== Creating run script ==="
cat > "$INSTALL_DIR/run_llm.sh" << 'EOF'
#!/usr/bin/env bash
INSTALL_DIR="$(cd "$(dirname "$0")" && pwd)"
MODEL="$INSTALL_DIR/models/mistral-7b-instruct-v0.2.Q4_K_M.gguf"
BINARY="$INSTALL_DIR/llama.cpp/build/bin/llama-server"

exec "$BINARY" \
    --model "$MODEL" \
    --ctx-size 4096 \
    --threads "$(nproc)" \
    --host 0.0.0.0 \
    --port 8080 \
    "$@"
EOF
chmod +x "$INSTALL_DIR/run_llm.sh"

echo ""
echo "=== Done ==="
echo "Run:           ./run_llm.sh"
echo "Web chat UI:   http://localhost:8080"
echo "OpenAI API:    http://localhost:8080/v1/chat/completions"