LLM-Server mit Open WebUI mit Ollama

Readme.md

Einrichtung eines Servers mit Nvidia-Hardware als LLM-Server

Treiber und Voraussetzungen

Für Ubuntu 24.04:

sudo apt update
sudo apt upgrade
sudo apt install nvidia-utils-580 nvidia-driver-580

Docker installieren und aktuellen Nutzer berechtigen:

curl -fsSL https://get.docker.com | sudo sh
sudo usermod -aG docker $USER

Nvidia-Repos einrichten

curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg \
  && curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \
    sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
    sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list

Nvidia-Toolkit für Container installieren:

sudo apt-get update
sudo apt install -y nvidia-container-toolkit

Toolkit in Docker aktivieren:

sudo nvidia-ctk runtime configure --runtime=docker
sudo systemctl restart docker

docker-compose-ingress.yml

# Variante mit Ingress-Router für HTTPS

services:
  openwebui:
    image: ghcr.io/open-webui/open-webui:main
    labels:
      - "traefik.http.routers.webui.rule=Host(`ai.example.org`)"
      - "traefik.http.routers.webui.tls.certResolver=default"
      - "traefik.http.routers.webui.tls=true"
      - "traefik.http.services.webui.loadbalancer.server.port=8080"
    volumes:
      - ./data/open-webui:/app/backend/data
  ollama:
    image: ollama/ollama:latest
    container_name: ollama
    volumes:
      - ./data/ollama:/root/.ollama
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    restart: unless-stopped

  traefik:
    image: traefik:v3
    command: --providers.docker
    restart: always
    ports:
      - 80:80
      - 443:443
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock:rw
      - ./data/traefik/static.yml:/etc/traefik/traefik.yml
      - ./data/traefik/dynamic.yml:/etc/traefik/dynamic/dynamic.yml
      - ./data/traefik/acme.json:/etc/traefik/acme/acme.json

docker-compose.yml

#Einfache Variante ohne Ingress-Router

services:
  openwebui:
    image: ghcr.io/open-webui/open-webui:main
    ports:
     - 8080:8080
    volumes:
      - ./data/open-webui:/app/backend/data

  ollama:
    image: ollama/ollama:latest
    container_name: ollama
    volumes:
      - ./data/ollama:/root/.ollama
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia

dynamic.yml

http:
  routers:
    https-redirect:
      rule: "HostRegexp(`{any:.*}`)"
      middlewares:
        - https-redirect
      service: redirect-all
  middlewares:
    https-redirect:
      redirectScheme:
        scheme: https
  services:
    redirect-all:
      loadBalancer:
        servers:
          - url: ""

static.yml

entryPoints:
  web:
    address: ":80"
  web-secure:
    address: ":443"

providers:
  docker:
    watch: true
  file:
    directory: /etc/traefik/dynamic
    watch: true
    filename: dynamic.yml

certificatesResolvers:
  default:
    acme:
      email: ihremail@example.org
      storage: /etc/traefik/acme/acme.json
      httpChallenge:
        entryPoint: web