klueska · September 10, 2026 09:20
diff --git a/Dockerfile.sandbox-gpu-test b/Dockerfile.sandbox-gpu-test
 # Reproduction image for the sandbox-gpu-cdi-support branch's real-hardware GPU tests.
 #
 # Stage 1 builds runsc (gVisor) from source instead of using the prebuilt release
 # binary from storage.googleapis.com/gvisor. Why: gVisor checks the *runtime* host's
 # page size against a Go constant baked in at *compile* time (unix.Getpagesize() vs.
 # hostarch.PageSize in runsc/cmd/sentry/sentrycmd/boot.go) and fatals on a mismatch in
 # either direction. The prebuilt aarch64 release binary is compiled for 4K pages only
 # (pkg/hostarch/hostarch_arm64.go's default), which hard-fails on ARM64 hosts whose
 # kernel boots with 64K pages (e.g. NVIDIA Grace/GB200 -- NVIDIA recommends 64K there
 # for HBM throughput, even though most distros default to 4K and the silicon supports
 # both): "FATAL ERROR: host page size (65536) does not match compiled page size (4096)".
 # Building with --define=pagesize=64k (added in google/gvisor@80bb741, 2026-09-01) fixes
 # that, but a binary built that way then fatals the *opposite* way on a 4K host. This
 # isn't published as a prebuilt artifact in any of gVisor's release/master/nightly
 # channels either way, so building from source is currently the only path to a
 # 64K-page-aware runsc at all.
 #
 # Rather than hardcode one choice, this Dockerfile detects the page size at build time
 # (`getconf PAGESIZE`, below) and passes --define=pagesize=64k only when it's 64K.
 # That's safe here specifically because `docker build` shares the host kernel with
 # wherever `docker build` itself runs (containers aren't VMs, and page size is a
 # kernel/hardware property, not something namespaced) -- so as long as you build this
 # image directly on the machine you intend to run it on (the expected usage, since GPU
 # access needs the real host's driver/devices anyway), the page size the build stage
 # observes is the page size the resulting binary needs to match. This does NOT hold if
 # you cross-build under emulation (e.g. `docker buildx build --platform`) for a
 # different machine than the one running the build -- don't do that for this image.
 # An ENTRYPOINT-based runtime dispatch (build both variants, symlink the right one at
 # container start) was considered instead, but doesn't work here: KubeRay sets its own
 # container `command`, which replaces the image's ENTRYPOINT entirely (confirmed
 # against a real KubeRay deployment -- the wrapper script silently never ran).
 #
 # Platform handling: TARGETARCH is populated automatically by BuildKit (including under
 # `docker buildx build --platform`, where it reflects the target, not the builder host).
 # Plain `docker build` on modern Docker (BuildKit-backed by default) also sets it from
 # the local host arch. We map amd64->x86_64/arm64->aarch64 for bazelisk's asset naming
 # and rely on gVisor's own BUILD files to select the right toolchain/output per target.
 FROM ubuntu:24.04 AS gvisor-builder
 ARG TARGETARCH
 ARG GVISOR_COMMIT=80bb741691be65cedb6688ac518bef3664af0fcc
 ARG BAZELISK_VERSION=v1.29.0

 # git/curl/python3: bazel + gvisor build scripts need them.
 # build-essential + clang + libbpf-dev: gvisor's non-Go bits (vdso, XDP eBPF progs) need a
 #   C/C++ toolchain and libbpf headers regardless of target arch.
 # gcc/g++-x86-64-linux-gnu + gcc/g++-aarch64-linux-gnu: runsc embeds a vdso blob for both
 #   amd64 and arm64 guests (containers can run either), so both cross-compilers are needed
 #   no matter which arch the build itself runs on -- an arm64 build host still needs the
 #   x86-64 one to produce the amd64 vdso.so genrule output, and an amd64 build host still
 #   needs the aarch64 one to produce the arm64 vdso.so genrule output.
 # libc6-dev-i386: gVisor's XDP eBPF host tools (tools/xdp/cmd/bpf/*.ebpf.c) compile some
 #   code as 32-bit, which needs 32-bit glibc headers (gnu/stubs-32.h) on an x86_64 build
 #   host. Conditional on the build host actually being amd64, not just harmlessly
 #   unconditional: arm64 hosts don't have an i386 variant to install at all, and
 #   `apt-get install` hard-fails ("Unable to locate package") rather than no-op-ing.
 RUN barch="${TARGETARCH:-$(dpkg --print-architecture)}"; \
    extra_pkgs=""; \
    case "$barch" in \
      amd64|x86_64) extra_pkgs="libc6-dev-i386" ;; \
      arm64|aarch64) extra_pkgs="" ;; \
      *) echo "unsupported arch: $barch" >&2; exit 1 ;; \
    esac; \
    apt-get update -qq && apt-get install -y -qq --no-install-recommends \
      git ca-certificates curl python3 build-essential clang libbpf-dev \
      gcc-x86-64-linux-gnu g++-x86-64-linux-gnu \
      gcc-aarch64-linux-gnu g++-aarch64-linux-gnu \
      $extra_pkgs \
    && rm -rf /var/lib/apt/lists/*

 RUN barch="${TARGETARCH:-$(dpkg --print-architecture)}"; \
    case "$barch" in \
      amd64|x86_64) barch=amd64 ;; \
      arm64|aarch64) barch=arm64 ;; \
      *) echo "unsupported arch: $barch" >&2; exit 1 ;; \
    esac; \
    curl -fsSL -o /usr/local/bin/bazelisk \
      "https://github.com/bazelbuild/bazelisk/releases/download/${BAZELISK_VERSION}/bazelisk-linux-${barch}" \
    && chmod +x /usr/local/bin/bazelisk

 RUN git clone --depth 1 https://github.com/google/gvisor.git /gvisor \
    && cd /gvisor && git fetch --depth 1 origin "$GVISOR_COMMIT" && git checkout "$GVISOR_COMMIT"

 WORKDIR /gvisor
 RUN pagesize_define=""; \
    case "$(getconf PAGESIZE)" in \
      4096) pagesize_define="" ;; \
      65536) pagesize_define="--define=pagesize=64k" ;; \
      *) echo "unsupported page size: $(getconf PAGESIZE)" >&2; exit 1 ;; \
    esac; \
    USE_BAZEL_VERSION="$(cat .bazelversion)" bazelisk build -c opt $pagesize_define //runsc \
    && cp bazel-bin/runsc/runsc_/runsc /usr/local/bin/runsc \
    && chmod +x /usr/local/bin/runsc

 # Stage 2: stage this branch's changed .py files (+ the minimal ray/tests scaffolding
 # the sandbox test conftest chain needs) into a site-packages/ray/... shaped tree, from
 # the ray repo checkout itself -- build with `-f Dockerfile.sandbox-gpu-test <path to
 # the repo, checked out at sandbox-gpu-cdi-support>` as the build context so this stage
 # can see .git and diff against the canonical ray-project/ray repo's master, fetched
 # directly by URL below rather than via a remote named in the checkout -- different
 # checkouts name their remotes differently (or may not have ray-project/ray fetched
 # under any remote at all), so resolving the base ref by remote name isn't portable.
 # Needs BRANCH_REF reachable in that checkout's git history (default: HEAD, i.e.
 # whatever commit is checked out); override with --build-arg if your checkout differs.
 FROM alpine:3.20 AS overlay-stager
 RUN apk add --no-cache git bash
 ARG BRANCH_REF=HEAD
 COPY . /src
 WORKDIR /src
 RUN set -e; \
    git remote add _upstream https://github.com/ray-project/ray.git; \
    git fetch _upstream master; \
    base="$(git merge-base _upstream/master "$BRANCH_REF")"; \
    mkdir -p /ray_overlay/ray; \
    for f in $(git diff --name-only "$base" "$BRANCH_REF" | grep '\.py$' | grep '^python/'); do \
      rel="${f#python/}"; mkdir -p "/ray_overlay/$(dirname "$rel")"; \
      cp "$f" "/ray_overlay/$rel"; \
    done; \
    mkdir -p /ray_overlay/ray/tests/accelerators; \
    cp python/ray/tests/conftest.py /ray_overlay/ray/tests/; \
    cp python/ray/tests/accelerators/mock_pynvml.py /ray_overlay/ray/tests/accelerators/; \
    cp python/ray/experimental/sandbox/tests/conftest.py \
       python/ray/experimental/sandbox/tests/__init__.py \
       /ray_overlay/ray/experimental/sandbox/tests/; \
    mkdir -p /ray_overlay/ray/experimental/sandbox/_internal; \
    cp python/ray/experimental/sandbox/_internal/image_utils.py \
       /ray_overlay/ray/experimental/sandbox/_internal/

 # Stage 3: overlay the staged tree onto the stock published ray image -- same
 # site-packages layout, no core rebuild needed since nothing outside pure Python changed.
 #
 # NOTE: this variant deliberately does NOT install nvidia-container-toolkit-base
 # (and has no `ar`/binutils either), to validate that ensure_nvidia_ctk
 # (conftest.py) can bootstrap it itself at test time.
 FROM rayproject/ray:2.58.0
 USER root
 # /usr/local/bin is root-owned and the container mostly runs as the non-root
 # `ray` user; runsc lives in a ray-owned dir on PATH instead. Baked in
 # directly at build time (not via an ENTRYPOINT wrapper) -- stage 1 already
 # resolves the right page-size variant at build time, and an ENTRYPOINT-based
 # runtime dispatch wouldn't work under KubeRay anyway (see the pagesize note
 # on stage 1 above).
 RUN mkdir -p /home/ray/bin && chown ray:users /home/ray/bin
 ENV PATH="/home/ray/bin:${PATH}"
 COPY --from=gvisor-builder --chown=ray:users /usr/local/bin/runsc /home/ray/bin/runsc
 COPY --from=overlay-stager --chown=ray:users /ray_overlay/ray/ /home/ray/anaconda3/lib/python3.10/site-packages/ray/
 RUN /home/ray/anaconda3/bin/pip install --no-cache-dir pytest pyyaml
 USER ray
 ENV TEST_SANDBOX=1
 ENV RAY_SANDBOX_IGNORE_CGROUPS=1
	# Reproduction image for the sandbox-gpu-cdi-support branch's real-hardware GPU tests.
	#
	# Stage 1 builds runsc (gVisor) from source instead of using the prebuilt release
	# binary from storage.googleapis.com/gvisor. Why: gVisor checks the runtime host's
	# page size against a Go constant baked in at compile time (unix.Getpagesize() vs.
	# hostarch.PageSize in runsc/cmd/sentry/sentrycmd/boot.go) and fatals on a mismatch in
	# either direction. The prebuilt aarch64 release binary is compiled for 4K pages only
	# (pkg/hostarch/hostarch_arm64.go's default), which hard-fails on ARM64 hosts whose
	# kernel boots with 64K pages (e.g. NVIDIA Grace/GB200 -- NVIDIA recommends 64K there
	# for HBM throughput, even though most distros default to 4K and the silicon supports
	# both): "FATAL ERROR: host page size (65536) does not match compiled page size (4096)".
	# Building with --define=pagesize=64k (added in google/gvisor@80bb741, 2026-09-01) fixes
	# that, but a binary built that way then fatals the opposite way on a 4K host. This
	# isn't published as a prebuilt artifact in any of gVisor's release/master/nightly
	# channels either way, so building from source is currently the only path to a
	# 64K-page-aware runsc at all.
	#
	# Rather than hardcode one choice, this Dockerfile detects the page size at build time
	# (`getconf PAGESIZE`, below) and passes --define=pagesize=64k only when it's 64K.
	# That's safe here specifically because `docker build` shares the host kernel with
	# wherever `docker build` itself runs (containers aren't VMs, and page size is a
	# kernel/hardware property, not something namespaced) -- so as long as you build this
	# image directly on the machine you intend to run it on (the expected usage, since GPU
	# access needs the real host's driver/devices anyway), the page size the build stage
	# observes is the page size the resulting binary needs to match. This does NOT hold if
	# you cross-build under emulation (e.g. `docker buildx build --platform`) for a
	# different machine than the one running the build -- don't do that for this image.
	# An ENTRYPOINT-based runtime dispatch (build both variants, symlink the right one at
	# container start) was considered instead, but doesn't work here: KubeRay sets its own
	# container `command`, which replaces the image's ENTRYPOINT entirely (confirmed
	# against a real KubeRay deployment -- the wrapper script silently never ran).
	#
	# Platform handling: TARGETARCH is populated automatically by BuildKit (including under
	# `docker buildx build --platform`, where it reflects the target, not the builder host).
	# Plain `docker build` on modern Docker (BuildKit-backed by default) also sets it from
	# the local host arch. We map amd64->x86_64/arm64->aarch64 for bazelisk's asset naming
	# and rely on gVisor's own BUILD files to select the right toolchain/output per target.
	FROM ubuntu:24.04 AS gvisor-builder
	ARG TARGETARCH
	ARG GVISOR_COMMIT=80bb741691be65cedb6688ac518bef3664af0fcc
	ARG BAZELISK_VERSION=v1.29.0

	# git/curl/python3: bazel + gvisor build scripts need them.
	# build-essential + clang + libbpf-dev: gvisor's non-Go bits (vdso, XDP eBPF progs) need a
	# C/C++ toolchain and libbpf headers regardless of target arch.
	# gcc/g++-x86-64-linux-gnu + gcc/g++-aarch64-linux-gnu: runsc embeds a vdso blob for both
	# amd64 and arm64 guests (containers can run either), so both cross-compilers are needed
	# no matter which arch the build itself runs on -- an arm64 build host still needs the
	# x86-64 one to produce the amd64 vdso.so genrule output, and an amd64 build host still
	# needs the aarch64 one to produce the arm64 vdso.so genrule output.
	# libc6-dev-i386: gVisor's XDP eBPF host tools (tools/xdp/cmd/bpf/*.ebpf.c) compile some
	# code as 32-bit, which needs 32-bit glibc headers (gnu/stubs-32.h) on an x86_64 build
	# host. Conditional on the build host actually being amd64, not just harmlessly
	# unconditional: arm64 hosts don't have an i386 variant to install at all, and
	# `apt-get install` hard-fails ("Unable to locate package") rather than no-op-ing.
	RUN barch="${TARGETARCH:-$(dpkg --print-architecture)}"; \
	extra_pkgs=""; \
	case "$barch" in \
	amd64\|x86_64) extra_pkgs="libc6-dev-i386" ;; \
	arm64\|aarch64) extra_pkgs="" ;; \
	*) echo "unsupported arch: $barch" >&2; exit 1 ;; \
	esac; \
	apt-get update -qq && apt-get install -y -qq --no-install-recommends \
	git ca-certificates curl python3 build-essential clang libbpf-dev \
	gcc-x86-64-linux-gnu g++-x86-64-linux-gnu \
	gcc-aarch64-linux-gnu g++-aarch64-linux-gnu \
	$extra_pkgs \
	&& rm -rf /var/lib/apt/lists/*

	RUN barch="${TARGETARCH:-$(dpkg --print-architecture)}"; \
	case "$barch" in \
	amd64\|x86_64) barch=amd64 ;; \
	arm64\|aarch64) barch=arm64 ;; \
	*) echo "unsupported arch: $barch" >&2; exit 1 ;; \
	esac; \
	curl -fsSL -o /usr/local/bin/bazelisk \
	"https://github.com/bazelbuild/bazelisk/releases/download/${BAZELISK_VERSION}/bazelisk-linux-${barch}" \
	&& chmod +x /usr/local/bin/bazelisk

	RUN git clone --depth 1 https://github.com/google/gvisor.git /gvisor \
	&& cd /gvisor && git fetch --depth 1 origin "$GVISOR_COMMIT" && git checkout "$GVISOR_COMMIT"

	WORKDIR /gvisor
	RUN pagesize_define=""; \
	case "$(getconf PAGESIZE)" in \
	4096) pagesize_define="" ;; \
	65536) pagesize_define="--define=pagesize=64k" ;; \
	*) echo "unsupported page size: $(getconf PAGESIZE)" >&2; exit 1 ;; \
	esac; \
	USE_BAZEL_VERSION="$(cat .bazelversion)" bazelisk build -c opt $pagesize_define //runsc \
	&& cp bazel-bin/runsc/runsc_/runsc /usr/local/bin/runsc \
	&& chmod +x /usr/local/bin/runsc

	# Stage 2: stage this branch's changed .py files (+ the minimal ray/tests scaffolding
	# the sandbox test conftest chain needs) into a site-packages/ray/... shaped tree, from
	# the ray repo checkout itself -- build with `-f Dockerfile.sandbox-gpu-test <path to
	# the repo, checked out at sandbox-gpu-cdi-support>` as the build context so this stage
	# can see .git and diff against the canonical ray-project/ray repo's master, fetched
	# directly by URL below rather than via a remote named in the checkout -- different
	# checkouts name their remotes differently (or may not have ray-project/ray fetched
	# under any remote at all), so resolving the base ref by remote name isn't portable.
	# Needs BRANCH_REF reachable in that checkout's git history (default: HEAD, i.e.
	# whatever commit is checked out); override with --build-arg if your checkout differs.
	FROM alpine:3.20 AS overlay-stager
	RUN apk add --no-cache git bash
	ARG BRANCH_REF=HEAD
	COPY . /src
	WORKDIR /src
	RUN set -e; \
	git remote add _upstream https://github.com/ray-project/ray.git; \
	git fetch _upstream master; \
	base="$(git merge-base _upstream/master "$BRANCH_REF")"; \
	mkdir -p /ray_overlay/ray; \
	for f in $(git diff --name-only "$base" "$BRANCH_REF" \| grep '\.py$' \| grep '^python/'); do \
	rel="${f#python/}"; mkdir -p "/ray_overlay/$(dirname "$rel")"; \
	cp "$f" "/ray_overlay/$rel"; \
	done; \
	mkdir -p /ray_overlay/ray/tests/accelerators; \
	cp python/ray/tests/conftest.py /ray_overlay/ray/tests/; \
	cp python/ray/tests/accelerators/mock_pynvml.py /ray_overlay/ray/tests/accelerators/; \
	cp python/ray/experimental/sandbox/tests/conftest.py \
	python/ray/experimental/sandbox/tests/__init__.py \
	/ray_overlay/ray/experimental/sandbox/tests/; \
	mkdir -p /ray_overlay/ray/experimental/sandbox/_internal; \
	cp python/ray/experimental/sandbox/_internal/image_utils.py \
	/ray_overlay/ray/experimental/sandbox/_internal/

	# Stage 3: overlay the staged tree onto the stock published ray image -- same
	# site-packages layout, no core rebuild needed since nothing outside pure Python changed.
	#
	# NOTE: this variant deliberately does NOT install nvidia-container-toolkit-base
	# (and has no `ar`/binutils either), to validate that ensure_nvidia_ctk
	# (conftest.py) can bootstrap it itself at test time.
	FROM rayproject/ray:2.58.0
	USER root
	# /usr/local/bin is root-owned and the container mostly runs as the non-root
	# `ray` user; runsc lives in a ray-owned dir on PATH instead. Baked in
	# directly at build time (not via an ENTRYPOINT wrapper) -- stage 1 already
	# resolves the right page-size variant at build time, and an ENTRYPOINT-based
	# runtime dispatch wouldn't work under KubeRay anyway (see the pagesize note
	# on stage 1 above).
	RUN mkdir -p /home/ray/bin && chown ray:users /home/ray/bin
	ENV PATH="/home/ray/bin:${PATH}"
	COPY --from=gvisor-builder --chown=ray:users /usr/local/bin/runsc /home/ray/bin/runsc
	COPY --from=overlay-stager --chown=ray:users /ray_overlay/ray/ /home/ray/anaconda3/lib/python3.10/site-packages/ray/
	RUN /home/ray/anaconda3/bin/pip install --no-cache-dir pytest pyyaml
	USER ray
	ENV TEST_SANDBOX=1
	ENV RAY_SANDBOX_IGNORE_CGROUPS=1
No results found