Ternary-Bonsai-2-27B-PQ2_0 on amd RX 6700 XT

compose.yaml

services:
  bonsai:
    image: rocm/dev-ubuntu-24.04@sha256:86e11093b4a7ec2a79b1b6701d10e840a6994f21c7e05929b51eb9be361c683a
    container_name: bonsai-2-27b
    restart: unless-stopped
    devices:
      - /dev/kfd:/dev/kfd
      - /dev/dri:/dev/dri
    environment:
      HSA_OVERRIDE_GFX_VERSION: 10.3.0
      LD_LIBRARY_PATH: /runtime:/opt/rocm/lib
    volumes:
      - ./runtime:/runtime:ro
      - ./models:/models:ro
    working_dir: /runtime
    ports:
      - 127.0.0.1:8331:8331
    command:
      - ./llama-server
      - -m
      - /models/Ternary-Bonsai-2-27B-PQ2_0.gguf
      - --alias
      - bonsai-2-27b
      - --host
      - 0.0.0.0
      - --port
      - "8331"
      - -ngl
      - "99"
      - --flash-attn
      - "off"
      - -c
      - "20000"
      - -b
      - "512"
      - -ub
      - "512"
      - --parallel
      - "1"
    healthcheck:
      test: [CMD-SHELL, "curl -fsS http://127.0.0.1:8331/health >/dev/null"]
      interval: 10s
      timeout: 3s
      retries: 12
      start_period: 30s