Launch llama.cpp, qwen and muse-glimmer

README.md

Usage:

./llama_build.sh rocm

./qwen-3.6_27b.32.sh rocm "/tmp/system_prompt.txt" "/tmp/prompt.txt"
# ./qwen-3.6_27b.16.sh rocm "/tmp/system_prompt.txt" "/tmp/prompt.txt"
# ./qwen-3.6_35b-a3b.32.6.sh rocm "/tmp/system_prompt.txt" "/tmp/prompt.txt"
# ./qwen-3.6_35b-a3b.64.6.sh rocm "/tmp/system_prompt.txt" "/tmp/prompt.txt"

./muse-glimmer_30b.32.sh rocm "/tmp/system_prompt.txt" "/tmp/prompt.txt"
# ./muse-glimmer_30b.16.sh rocm "/tmp/system_prompt.txt" "/tmp/prompt.txt"

Qwen results on dual 9060 XT 16 GB

Input tg = 680-720 t/s
Output tg = 35-40 t/s

Muse glimmer on dual 9060 XT 16 GB

Input tg = 590-670 t/s
Output tg = 33-37 t/s

llama_build.sh

#!/usr/bin/env bash
set -euo pipefail

MODE="${1:-cuda}"
ROCM_DOCKER_ARCH="${2:-gfx1100,gfx1101,gfx1102,gfx1200,gfx1201}"

case "$MODE" in
  cpu|cuda|rocm)
    ;;
  *)
    echo "Usage: $0 {cpu|cuda|rocm} [ROCM_DOCKER_ARCH]" >&2
    exit 2
    ;;
esac

GITHUB_REPOSITORY="ggml-org/llama.cpp"
GITHUB_REMOTE="git@github.com:${GITHUB_REPOSITORY}.git"
REPOSITORY_PATH="${HOME}/workspace/llama.cpp"
PROJECT="llama-server"
IMAGE="local/llama.cpp:server-${MODE}"
DOCKERFILE_RELATIVE_PATH=".devops/${MODE}.Dockerfile"
BUILD_TARGET="server"
CUDA_UBUNTU_VERSION="24.04"
CUDA_VERSION="12.9.2"
CUDA_DEV_CONTAINER="docker.io/nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${CUDA_UBUNTU_VERSION}"
CUDA_RUN_CONTAINER="docker.io/nvidia/cuda:${CUDA_VERSION}-runtime-ubuntu${CUDA_UBUNTU_VERSION}"
ROCM_UBUNTU_VERSION="26.04"
ROCM_VERSION="7.14.0"
ROCM_DEV_CONTAINER="docker.io/rocm/dev-ubuntu-${ROCM_UBUNTU_VERSION}:${ROCM_VERSION}-full"
ROCM_LIBRARY_PATH="/opt/rocm/core-${ROCM_VERSION%.*}/lib"
ROCM_LIBRARY_ENV="ENV LD_LIBRARY_PATH=${ROCM_LIBRARY_PATH}"

git init --quiet "$REPOSITORY_PATH"
git -C "$REPOSITORY_PATH" config --replace-all remote.origin.url "$GITHUB_REMOTE"

LATEST_TAG="$(
  gh release view \
    --repo "$GITHUB_REPOSITORY" \
    --json "tagName" \
    --jq ".tagName"
)"

git \
  -C "$REPOSITORY_PATH" \
  fetch \
  --depth "1" \
  --no-tags \
  "origin" \
  "refs/tags/${LATEST_TAG}"

SOURCE_COMMIT="$(git -C "$REPOSITORY_PATH" rev-parse "FETCH_HEAD^{commit}")"
BUILD_KEY="${LATEST_TAG}:${SOURCE_COMMIT}:${MODE}:${DOCKERFILE_RELATIVE_PATH}:${BUILD_TARGET}"
MODE_BUILD_ARGS=()

if [[ "$MODE" == "cuda" ]]; then
  MODE_BUILD_ARGS+=(
    --build-arg "BASE_CUDA_DEV_CONTAINER=${CUDA_DEV_CONTAINER}"
    --build-arg "BASE_CUDA_RUN_CONTAINER=${CUDA_RUN_CONTAINER}"
  )
  BUILD_KEY="${BUILD_KEY}:${CUDA_DEV_CONTAINER}:${CUDA_RUN_CONTAINER}"
elif [[ "$MODE" == "rocm" ]]; then
  ROCM_BUILD_ARG="ROCM_DOCKER_ARCH=${ROCM_DOCKER_ARCH//,/;}"
  MODE_BUILD_ARGS+=(
    --build-arg "$ROCM_BUILD_ARG"
    --build-arg "BASE_ROCM_DEV_CONTAINER=${ROCM_DEV_CONTAINER}"
  )
  RCCL_CMAKE_OPTION="-DGGML_HIP_RCCL=ON"
  BUILD_KEY="${BUILD_KEY}:${ROCM_BUILD_ARG}:${ROCM_DEV_CONTAINER}:${RCCL_CMAKE_OPTION}"
fi

CURRENT_BUILD_KEY="$(
  docker image inspect \
    --format '{{ index .Config.Labels "local.build-key" }}' \
    "$IMAGE" \
    2> /dev/null || true
)"

if [[ "$CURRENT_BUILD_KEY" != "$BUILD_KEY" ]]; then
  git -C "$REPOSITORY_PATH" checkout --force --detach "$SOURCE_COMMIT"
  git -C "$REPOSITORY_PATH" clean -dffx

  DOCKERFILE_PATH="${REPOSITORY_PATH}/${DOCKERFILE_RELATIVE_PATH}"

  if [[ "$MODE" == "rocm" ]]; then
    sed -i \
      "/FROM \${BASE_ROCM_DEV_CONTAINER} AS base/a ${ROCM_LIBRARY_ENV}" \
      "$DOCKERFILE_PATH"

    if ! grep -qF -- "$RCCL_CMAKE_OPTION" "$DOCKERFILE_PATH"; then
      sed -i '/-DGGML_HIP=ON \\/a\  -DGGML_HIP_RCCL=ON \\' "$DOCKERFILE_PATH"
    fi

    if ! grep -qF -- "$RCCL_CMAKE_OPTION" "$DOCKERFILE_PATH"; then
      echo "Failed to enable RCCL in ${DOCKERFILE_PATH}" >&2
      exit 1
    fi

    if ! grep -qFx -- "$ROCM_LIBRARY_ENV" "$DOCKERFILE_PATH"; then
      echo "Failed to configure ROCm library path in ${DOCKERFILE_PATH}" >&2
      exit 1
    fi
  fi

  BUILD_ARGS=(
    --file "$DOCKERFILE_PATH"
    --target "$BUILD_TARGET"
    --tag "$IMAGE"
    --label "local.project=${PROJECT}"
    --label "local.source-ref=${LATEST_TAG}"
    --label "local.source-commit=${SOURCE_COMMIT}"
    --label "local.build-key=${BUILD_KEY}"
    --progress "plain"
    "${MODE_BUILD_ARGS[@]}"
  )

  DOCKER_BUILDKIT=1 docker build "${BUILD_ARGS[@]}" "$REPOSITORY_PATH"
fi

docker system prune --all --force --volumes --filter "label!=local.build-key=${BUILD_KEY}"

llama_run.sh

#!/usr/bin/env bash
set -euo pipefail

usage() {
  echo "Usage: $0 {cpu|cuda|rocm} REPOSITORY_NAME SYSTEM_PROMPT_FILE_PATH PROMPT_FILE_PATH CONTEXT_SIZE TEMPERATURE TOP_P TOP_K MIN_P PRESENCE_PENALTY REPETITION_PENALTY [LLAMA_SERVER_ARGS...]" >&2
  exit 2
}

if [[ "$#" -lt 11 ]]; then
  usage
fi

MODE="$1"
REPOSITORY_NAME="$2"
SYSTEM_PROMPT_FILE_PATH="$3"
PROMPT_FILE_PATH="$4"
CONTEXT_SIZE="$5"
TEMPERATURE="$6"
TOP_P="$7"
TOP_K="$8"
MIN_P="$9"
PRESENCE_PENALTY="${10}"
REPETITION_PENALTY="${11}"
shift 11

CONTAINER_NAME="llama-server"
SERVER_PORT="9090"
SERVER_URL="http://127.0.0.1:${SERVER_PORT}"
META_FILE_PATH="/tmp/${CONTAINER_NAME}.txt"
CACHE_PATH="${HOME}/workspace/llama.cpp.data"
SLEEP_DURATION="1"

DOCKER_ARGS=()
SERVER_ARGS=(
  --hf-repo "$REPOSITORY_NAME"
  --alias "local"
  --host "127.0.0.1"
  --port "$SERVER_PORT"
  --parallel "1"
  --kv-unified
  --fit "on"
  --fit-target "32"
  --batch-size "512"
  --ubatch-size "256"
)

case "$MODE" in
  cpu)
    SERVER_ARGS+=(
      --device "none"
      --fit "off"
    )
    ;;
  cuda)
    DOCKER_ARGS+=(
      --gpus "all"
    )
    ;;
  rocm)
    DOCKER_ARGS+=(
      --device "/dev/dri"
      --device "/dev/kfd"
      --group-add "video"
      --security-opt "seccomp=unconfined"
    )
    ;;
  *)
    usage
    ;;
esac

IMAGE="local/llama.cpp:server-${MODE}"
IMAGE_ID="$(docker image inspect --format '{{.Id}}' "$IMAGE")"
GPU_COUNT="0"

if [[ "$MODE" != "cpu" ]]; then
  GPU_COUNT="$(
    docker run \
      --rm \
      "${DOCKER_ARGS[@]}" \
      "$IMAGE" \
      --list-devices \
      2>&1 |
      awk '
        $1 ~ /^(CUDA|ROCm)[0-9]+:$/ {
          count++
        }

        END {
          print count + 0
        }
      '
  )"

  if (( GPU_COUNT == 0 )); then
    echo "No ${MODE} GPUs detected" >&2
    exit 1
  fi

  if (( GPU_COUNT > 1 )); then
    DOCKER_ARGS+=(
      --ipc "host"
    )
  fi
fi

SERVER_ARGS+=(
  --no-mmproj
  --no-escape
  --log-verbosity "3"
  --ctx-size "$CONTEXT_SIZE"
  --no-context-shift
  "$@"
)

server_meta() {
  printf '%s\n' \
    "$MODE" \
    "$IMAGE_ID" \
    "$GPU_COUNT" \
    "${DOCKER_ARGS[@]}" \
    "${SERVER_ARGS[@]}"
}

server_running() {
  [[ "$(
    docker container inspect \
      --format '{{.State.Running}}' \
      "$CONTAINER_NAME" \
      2> /dev/null ||
      true
  )" == "true" ]]
}

if ! server_running || ! cmp -s "$META_FILE_PATH" <(server_meta); then
  docker rm -f "$CONTAINER_NAME" > /dev/null 2>&1 || :
  rm -f -- "$META_FILE_PATH"

  mkdir -p "$CACHE_PATH"

  docker run \
    --detach \
    --rm \
    --network "host" \
    "${DOCKER_ARGS[@]}" \
    --volume "${CACHE_PATH}:/root/.cache" \
    --name "$CONTAINER_NAME" \
    "$IMAGE" \
    "${SERVER_ARGS[@]}" \
    > /dev/null

  server_meta > "$META_FILE_PATH"
fi

until server_running && curl \
  --fail \
  --silent \
  "${SERVER_URL}/health" \
  > /dev/null 2>&1
do
  if ! server_running; then
    echo "${CONTAINER_NAME} exited before becoming healthy" >&2
    exit 1
  fi

  sleep "$SLEEP_DURATION"
done

# echo "> System prompt:" >&2
# cat "$SYSTEM_PROMPT_FILE_PATH" >&2
# echo "----" >&2
# echo >&2

echo "> User prompt:" >&2
cat "$PROMPT_FILE_PATH" >&2
echo "----" >&2
echo >&2

jq \
  -n \
  --rawfile system_prompt "$SYSTEM_PROMPT_FILE_PATH" \
  --rawfile prompt "$PROMPT_FILE_PATH" \
  --argjson temperature "$TEMPERATURE" \
  --argjson top_p "$TOP_P" \
  --argjson top_k "$TOP_K" \
  --argjson min_p "$MIN_P" \
  --argjson presence_penalty "$PRESENCE_PENALTY" \
  --argjson repetition_penalty "$REPETITION_PENALTY" \
  '{
    model: "local",
    temperature: $temperature,
    top_p: $top_p,
    top_k: $top_k,
    min_p: $min_p,
    presence_penalty: $presence_penalty,
    repeat_penalty: $repetition_penalty,
    messages: [
      {
        role: "system",
        content: $system_prompt
      },
      {
        role: "user",
        content: $prompt
      }
    ],
    stream: true
  }' |
curl \
  --fail-with-body \
  --no-buffer \
  --silent \
  --show-error \
  --header "Content-Type: application/json" \
  --data-binary @- \
  "${SERVER_URL}/v1/chat/completions" |
jq \
  --raw-input \
  --raw-output \
  --join-output \
  --unbuffered \
  '
    select(startswith("data:"))
    | sub("^data: *"; "")
    | sub("\r$"; "")
    | select(. != "[DONE]")
    | fromjson?
    | .choices[]?.delta.content // empty
  '

printf '\n'

llama_stop.sh

#!/usr/bin/env bash
set -euo pipefail

CONTAINER_NAME="llama-server"
META_FILE_PATH="/tmp/${CONTAINER_NAME}.txt"

docker rm -f "$CONTAINER_NAME" > /dev/null 2>&1 || :
rm -f -- "$META_FILE_PATH"

muse-glimmer_30b.16.sh

#!/usr/bin/env bash
set -euo pipefail

DIR_PATH="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
MAIN_PATH="${DIR_PATH}/main.sh"

MODE="$1"
SYSTEM_PROMPT_FILE_PATH="$2"
PROMPT_FILE_PATH="$3"
shift 3

# Setup for single RX 7600 XT 16GB.
REPOSITORY_NAME="unsloth/Muse-Glimmer-30B-GGUF:Q3_K_XL"
CONTEXT_SIZE=60160

exec "$MAIN_PATH" \
  "$MODE" \
  "$REPOSITORY_NAME" \
  "$SYSTEM_PROMPT_FILE_PATH" \
  "$PROMPT_FILE_PATH" \
  "$CONTEXT_SIZE" \
  --spec-type "draft-dflash" \
  --spec-draft-n-max "3" \
  --fit "off" \
  --n-gpu-layers "all" \
  "$@"

muse-glimmer_30b.32.sh

#!/usr/bin/env bash
set -euo pipefail

DIR_PATH="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
MAIN_PATH="${DIR_PATH}/main.sh"

MODE="$1"
SYSTEM_PROMPT_FILE_PATH="$2"
PROMPT_FILE_PATH="$3"
shift 3

# Setup for dual RX 9060 XT 16GB.
REPOSITORY_NAME="unsloth/Muse-Glimmer-30B-GGUF:Q8_0"
CONTEXT_SIZE=60160

exec "$MAIN_PATH" \
  "$MODE" \
  "$REPOSITORY_NAME" \
  "$SYSTEM_PROMPT_FILE_PATH" \
  "$PROMPT_FILE_PATH" \
  "$CONTEXT_SIZE" \
  --spec-type "draft-dflash" \
  --spec-draft-n-max "3" \
  --split-mode "tensor" \
  --fit "off" \
  --n-gpu-layers "all" \
  "$@"

muse-glimmer_main.sh

#!/usr/bin/env bash
set -euo pipefail

MODE="$1"
REPOSITORY_NAME="$2"
SYSTEM_PROMPT_FILE_PATH="$3"
PROMPT_FILE_PATH="$4"
CONTEXT_SIZE="$5"
shift 5

DIR_PATH="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
RUN_PATH="${DIR_PATH}/llama_run.sh"

TEMPERATURE="1.0"
TOP_P="0.95"
TOP_K="64"
MIN_P="0.05"
PRESENCE_PENALTY="0.0"
REPETITION_PENALTY="1.0"

exec "$RUN_PATH" \
  "$MODE" \
  "$REPOSITORY_NAME" \
  "$SYSTEM_PROMPT_FILE_PATH" \
  "$PROMPT_FILE_PATH" \
  "$CONTEXT_SIZE" \
  "$TEMPERATURE" \
  "$TOP_P" \
  "$TOP_K" \
  "$MIN_P" \
  "$PRESENCE_PENALTY" \
  "$REPETITION_PENALTY" \
  "$@"

qwen-3.6_27b.16.sh

#!/usr/bin/env bash
set -euo pipefail

DIR_PATH="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
MAIN_PATH="${DIR_PATH}/main.sh"

MODE="$1"
SYSTEM_PROMPT_FILE_PATH="$2"
PROMPT_FILE_PATH="$3"
shift 3

# Setup for single RX 7600 XT 16GB.
REPOSITORY_NAME="unsloth/Qwen3.6-27B-MTP-GGUF:Q3_K_XL"

# Max possible context 66304, offset 6144.
CONTEXT_SIZE=60160

exec "$MAIN_PATH" \
  "$MODE" \
  "$REPOSITORY_NAME" \
  "$SYSTEM_PROMPT_FILE_PATH" \
  "$PROMPT_FILE_PATH" \
  "$CONTEXT_SIZE" \
  --flash-attn "on" \
  --cache-type-k "q8_0" \
  --cache-type-v "q8_0" \
  --spec-type "draft-mtp" \
  --spec-draft-n-max "3" \
  --fit "off" \
  --n-gpu-layers "all" \
  "$@"

qwen-3.6_27b.32.sh

#!/usr/bin/env bash
set -euo pipefail

DIR_PATH="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
MAIN_PATH="${DIR_PATH}/main.sh"

MODE="$1"
SYSTEM_PROMPT_FILE_PATH="$2"
PROMPT_FILE_PATH="$3"
shift 3

# Setup for dual RX 9060 XT 16GB.
REPOSITORY_NAME="unsloth/Qwen3.6-27B-MTP-GGUF:Q8_0"

# Max possible context 66304, offset 6144.
CONTEXT_SIZE=60160

exec "$MAIN_PATH" \
  "$MODE" \
  "$REPOSITORY_NAME" \
  "$SYSTEM_PROMPT_FILE_PATH" \
  "$PROMPT_FILE_PATH" \
  "$CONTEXT_SIZE" \
  --spec-type "draft-mtp" \
  --spec-draft-n-max "3" \
  --split-mode "tensor" \
  --fit "off" \
  --n-gpu-layers "all" \
  "$@"

qwen-3.6_35b-a3b.32.6.sh

#!/usr/bin/env bash
set -euo pipefail

DIR_PATH="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
MAIN_PATH="${DIR_PATH}/main.sh"

MODE="$1"
SYSTEM_PROMPT_FILE_PATH="$2"
PROMPT_FILE_PATH="$3"
shift 3

REPOSITORY_NAME="unsloth/Qwen3.6-35B-A3B-GGUF:Q5_K_XL"
CONTEXT_SIZE=60160

exec "$MAIN_PATH" \
  "$MODE" \
  "$REPOSITORY_NAME" \
  "$SYSTEM_PROMPT_FILE_PATH" \
  "$PROMPT_FILE_PATH" \
  "$CONTEXT_SIZE" \
  --cpu-moe \
  "$@"

qwen-3.6_35b-a3b.64.6.sh

#!/usr/bin/env bash
set -euo pipefail

DIR_PATH="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
MAIN_PATH="${DIR_PATH}/main.sh"

MODE="$1"
SYSTEM_PROMPT_FILE_PATH="$2"
PROMPT_FILE_PATH="$3"
shift 3

REPOSITORY_NAME="unsloth/Qwen3.6-35B-A3B-GGUF:Q8_K_XL"
CONTEXT_SIZE=60160

exec "$MAIN_PATH" \
  "$MODE" \
  "$REPOSITORY_NAME" \
  "$SYSTEM_PROMPT_FILE_PATH" \
  "$PROMPT_FILE_PATH" \
  "$CONTEXT_SIZE" \
  --cpu-moe \
  "$@"

qwen-3.6_main.sh

#!/usr/bin/env bash
set -euo pipefail

MODE="$1"
REPOSITORY_NAME="$2"
SYSTEM_PROMPT_FILE_PATH="$3"
PROMPT_FILE_PATH="$4"
CONTEXT_SIZE="$5"
shift 5

DIR_PATH="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
RUN_PATH="${DIR_PATH}/llama_run.sh"

TEMPERATURE="1.0"
TOP_P="0.95"
TOP_K="20"
MIN_P="0.0"
PRESENCE_PENALTY="0.0"
REPETITION_PENALTY="1.05"

exec "$RUN_PATH" \
  "$MODE" \
  "$REPOSITORY_NAME" \
  "$SYSTEM_PROMPT_FILE_PATH" \
  "$PROMPT_FILE_PATH" \
  "$CONTEXT_SIZE" \
  "$TEMPERATURE" \
  "$TOP_P" \
  "$TOP_K" \
  "$MIN_P" \
  "$PRESENCE_PENALTY" \
  "$REPETITION_PENALTY" \
  "$@"