create a working directory. i use ~/Projects/aimodels
than create subdirectory for the model like mkdir ~/Projects/aimodels/huggingface.co/TheBloke
and cd ~/Projects/aimodels/huggingface.co/TheBloke
now download the model wget https://huggingface.co/TheBloke/Guanaco-13B-Uncensored-GGUF/resolve/main/guanaco-13b-uncensored.Q4_0.gguf
create docker-compose.yaml
version: '3.6'
services:
llama-cpp-api:
image: ghcr.io/abetlen/llama-cpp-python:latest
restart: on-failure