modal-qwen-server.py

# Deploy:  modal deploy server.py

import subprocess
import time
import urllib.request

import modal

# ── Config ───────────────────────────────────────────────────────────────────
APP_NAME     = "your-top-secret-server"
API_KEY      = "your-api-key-here"

MODEL_REPO   = "unsloth/Qwen3.6-27B-GGUF"
MODEL_FILE   = "Qwen3.6-27B-Q6_K.gguf"
MMPROJ_FILE  = "mmproj-BF16.gguf"
MODEL_PATH   = f"/models/{MODEL_FILE}"
MMPROJ_PATH  = f"/models/{MMPROJ_FILE}"
PORT         = 8000
CTX_SIZE     = 131072
N_PARALLEL   = 2
# ─────────────────────────────────────────────────────────────────────────────

volume = modal.Volume.from_name("qwen3-6-gguf-cache", create_if_missing=True)

image = (
    modal.Image.from_registry(
        "ghcr.io/ggml-org/llama.cpp:server-cuda",
        add_python="3.11",
    )
    .dockerfile_commands(["ENTRYPOINT []", "CMD []"])
    .pip_install("huggingface_hub[hf_transfer]")
    .env({"HF_HUB_ENABLE_HF_TRANSFER": "1"})
)

app = modal.App(APP_NAME, image=image)


@app.cls(
    gpu="L40S",
    volumes={"/models": volume},
    timeout=3600,
    scaledown_window=300,
    min_containers=0,
)
class LlamaCppServer:

    @modal.enter()
    def start(self):
        import os
        from huggingface_hub import hf_hub_download

        for filename, path in [(MODEL_FILE, MODEL_PATH), (MMPROJ_FILE, MMPROJ_PATH)]:
            if not os.path.exists(path):
                print(f"Downloading {filename} ...")
                hf_hub_download(
                    repo_id=MODEL_REPO,
                    filename=filename,
                    local_dir="/models",
                )
                print(f"{filename} downloaded.")
            else:
                print(f"{filename} found in cache -- skipping.")

        volume.commit()

        cmd = [
            "/app/llama-server",
            "-m",          MODEL_PATH,
            "--mmproj",    MMPROJ_PATH,       # vision projector
            "--host",      "0.0.0.0",
            "--port",      str(PORT),
            "-ngl",        "99",
            "-c",          str(CTX_SIZE),
            "--parallel",  str(N_PARALLEL),
            "--flash-attn", "on",
            "--cont-batching",
            "--temp", "1.0",
            "--top-p", "0.95",
            "--presence_penalty", "1.5",
            "--top-k", "20",
            "--chat_template_kwargs", '{"preserve_thinking": true}',
            "--api-key", API_KEY
        ]
        print("Starting llama-server ...")
        subprocess.Popen(cmd)

        print("Waiting for llama-server to be ready ...")
        deadline = time.time() + 300
        while time.time() < deadline:
            try:
                urllib.request.urlopen(f"http://localhost:{PORT}/health")
                print("llama-server is ready!")
                return
            except Exception:
                time.sleep(2)
        raise RuntimeError("llama-server failed to start within 5 minutes.")

    @modal.web_server(PORT, startup_timeout=360)
    def serve(self):
        pass