llama.cpp example

compose.yaml

services:
  llama-cpp:
    image: ghcr.io/ggml-org/llama.cpp:server-cuda
    container_name: llama-cpp
    restart: unless-stopped
    ports:
      - "8080:8080"
    volumes:
      - ./models:/models
    command:
      - "-m"
      - "/models/gemma-4-E2B-it-Q4_K_M.gguf"
      - "-ngl"
      - "99"
      - "-c"
      - "131072"
      - "--host"
      - "0.0.0.0"
    runtime: nvidia