mistral.rs Dockerfile + Cloud Build

cloudbuild.yaml

steps:
  - name: "gcr.io/cloud-builders/git"
    id: Clone repository
    args:
      - clone
      - https://github.com/EricLBuehler/mistral.rs.git
  - name: "gcr.io/cloud-builders/docker"
    id: build
    entrypoint: "bash"
    args:
      - -c
      - |
        docker buildx build --tag=${_IMAGE} .

images: ["${_IMAGE}"]

substitutions:
  _IMAGE: "us-central1-docker.pkg.dev/${PROJECT_ID}/ocr-test/mistralrs-minicpm-o"

options:
  dynamicSubstitutions: true
  machineType: "E2_HIGHCPU_32"

Dockerfile

FROM nvidia/cuda:12.4.1-devel-ubuntu22.04 AS builder

RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
    curl \
    libssl-dev \
    pkg-config \
    && rm -rf /var/lib/apt/lists/*

RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
ENV PATH="/root/.cargo/bin:${PATH}"
RUN rustup update nightly
RUN rustup default nightly

WORKDIR /mistralrs

COPY mistral.rs .

ARG CUDA_COMPUTE_CAP=80
ENV CUDA_COMPUTE_CAP=${CUDA_COMPUTE_CAP}
ARG FEATURES="cuda cudnn"
ENV RAYON_NUM_THREADS=4
RUN RUSTFLAGS="-Z threads=4" cargo build --release --workspace --exclude mistralrs-pyo3 --features "${FEATURES}"

FROM nvidia/cuda:12.4.1-runtime-ubuntu22.04 AS base

ENV HF_HOME=/hf_home \
    RAYON_NUM_THREADS=8 \ 
    LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH \
    RUST_BACKTRACE=1

# Run the script to create symlinks in /usr/local/cuda/lib64
RUN set -eux; \
    for lib in $(ls /usr/local/cuda/lib64); do \
        base=$(echo $lib | sed -r 's/(.+)\.so\..+/\1.so/'); \
        if [ "$lib" != "$base" ]; then \
            ln -sf "/usr/local/cuda/lib64/$lib" "/usr/local/cuda/lib64/$base"; \
        fi; \
    done

RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
    libomp-dev \
    ca-certificates \
    libssl-dev \
    curl \
    pkg-config \
    && rm -rf /var/lib/apt/lists/*


# Download the model so that it's included in the image
FROM python:3.13.2-slim-bookworm AS downloader

ENV HF_HOME=/hf_home
RUN pip install -U "huggingface_hub[cli]"
RUN huggingface-cli download openbmb/MiniCPM-o-2_6


FROM base

COPY --from=builder /mistralrs/target/release/mistralrs-bench /usr/local/bin/mistralrs-bench
RUN chmod +x /usr/local/bin/mistralrs-bench
COPY --from=builder /mistralrs/target/release/mistralrs-server /usr/local/bin/mistralrs-server
RUN chmod +x /usr/local/bin/mistralrs-server

COPY --from=downloader /hf_home /hf_home

ENTRYPOINT [ \
  "mistralrs-server", \
  "--port", "8000", \
  "--isq", "Q4K", \
  "vision-plain", \
  "-m", "openbmb/MiniCPM-o-2_6", \
  "-a", "minicpmo" \
]