Raspberry Pi 5 ML Setup (llama.cpp + Vulkan, etc)

README.md

Steps from https://github.com/ochafik/autopilot/tree/master

sudo apt-get update && sudo apt-get upgrade -y 
sudo apt-get install -y --no-install-recommends \
  code docker.io cmake screen vim mc fritzing inkscape \
  qemu-system-x86 qemu-user-static binfmt-support
sudo update-binfmts --enable qemu-x86_64

# Setup unattended upgrades:
sudo apt-get install unattended-upgrades
echo 'Unattended-Upgrade::Automatic-Reboot "true";' | sudo tee -a /etc/apt/apt.conf.d/50unattended-upgrades

# Setup a firewall:
sudo apt-get install ufw
sudo ufw allow ssh
yes | sudo ufw enable

# Setup IP banning for serial offenders:
sudo apt-get install fail2ban
cat /etc/fail2ban/jail.conf | \
  sed -E 's/^(maxretry *=).*/\1 3/g' | \
  sed -E 's/^(bantime *=).*/\1 -1/g' | \
  sudo tee /etc/fail2ban/jail.local
  
curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh"
bash Miniforge3-$(uname)-$(uname -m).sh

sudo apt install -y \
  cmake \
  vulkan-tools mesa-vulkan-drivers libvulkan-dev glslc \
  mesa-opencl-icd clinfo opencl-headers clblast-utils libclblast-dev \
  ocl-icd-opencl-dev ocl-icd-dev

GitHub

ssh-keygen -t rsa -C "olivier.chafik@gmail.com"

# Open github.com and setup public key access

Repos

mkdir -p ~/GitHub

# llama.cpp
git clone git@github.com:ochafik/llama.cpp.git
#+       if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64")
#+           # Rasberry Pi 5 (64-bit)
#+           add_compile_options(-mcpu=native -mtune=native -march=native)
#+       endif()
( cd llama.cpp && git checkout master && \
  cmake -B ../llama-build -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=1 -DCMAKE_BUILD_TYPE=Release && \
  ( cd ../llama-build && make -j4 ) )
( cd llama.cpp && git checkout master && \
  cmake -B ../llama-opencl-build -DLLAMA_CLBLAST=1 -DCMAKE_BUILD_TYPE=Release && \
  ( cd ../llama-opencl-build && make -j4 ) )
# Vulkan branch (https://github.com/ggerganov/llama.cpp/pull/2059)
( cd llama.cpp && git remote add 0cc4m_koboldcpp https://github.com/0cc4m/koboldcpp && git fetch 0cc4m_koboldcpp )
( cd llama.cpp && git checkout 0cc4m_koboldcpp/vulkan && \
  python ggml_vk_generate_shaders.py && \
  cmake -B ../llama-build-vulkan -DLLAMA_VULKAN=1 -DCMAKE_BUILD_TYPE=Release && \
  ( cd ../llama-build-vulkan && make -j4 ) )
  
# whisper.cpp
git clone git@github.com:ochafik/whisper.cpp.git
( cd whisper.cpp && git checkout master && \
  cmake -B ../whisper-build -DCMAKE_BUILD_TYPE=Release && \
  ( cd ../whisper-build && make -j4 ) )

Models

mkdir -p ~/Models
( cd ~/Models && wget https://huggingface.co/TheBloke/Mistral-7B-OpenOrca-GGUF/resolve/main/mistral-7b-openorca.Q2_K.gguf?download=true )
( cd ~/Models && wget https://huggingface.co/TheBloke/Mistral-7B-OpenOrca-GGUF/resolve/main/mistral-7b-openorca.Q5_K_M.gguf?download=true )

GGML_OPENCL_PLATFORM=rusticl GGML_OPENCL_DEVICE=1 \
  ~/GitHub/llama-opencl-build/bin/main -m ~/Models/mistral-7b-openorca.Q5_K_M.gguf \
    -p "Building a website can be done in 10 simple steps:\nStep 1:" \
    -n 100 -e \
    -ngl 20