| #!/bin/bash | |
| # Stop script on error | |
| set -e | |
| # Function to check if a command exists | |
| command_exists() { | |
| command -v "$1" >/dev/null 2>&1 | |
| } |
| ## Max Token | |
| python -m llama_cpp.server --model DeepSeek-R1-Distill-Qwen-1.5B-Q4_1.gguf --host 0.0.0.0 --n_threads 8 --n_batch 512 --n_gpu_layers 0 --n_ctx 2048 --mul_mat_q 1 | |
| # Balanced | |
| python -m llama_cpp.server --model DeepSeek-R1-Distill-Qwen-1.5B-Q4_1.gguf --host 0.0.0.0 --n_threads 8 --n_batch 32 --n_gpu_layers 0 --n_ctx 512 --mul_mat_q 1 --offload_kqv 1 | |
| ## VULKAN |
sudo apt update &&
sudo apt install -y
build-essential
dkms
wget https://repo.radeon.com/amdgpu-install/6.3.3/ubuntu/noble/amdgpu-install_6.3.60303-1_all.deb sudo apt install -y ./amdgpu-install_6.3.60303-1_all.deb
sudo amdgpu-install -y --usecase=rocm,mlsdk \
