qwen3-coder-next setup

dgd

apiVersion: nvidia.com/v1beta1
kind: DynamoGraphDeployment
metadata:
  annotations:
    meta.helm.sh/release-name: dynamo
    meta.helm.sh/release-namespace: dynamo
    nvidia.com/current-worker-hash: d9923307
    nvidia.com/current-worker-hash-v2: 5e2d36c2
    nvidia.com/dynamo-operator-origin-version: 1.2.0
  creationTimestamp: "2026-07-17T16:04:21Z"
  finalizers:
  - nvidia.com/finalizer
  generation: 3
  labels:
    app.kubernetes.io/instance: dynamo
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: dynamo
    app.kubernetes.io/version: 1.2.0-cuda13
    helm.sh/chart: dynamo-0.1.0-rc24
  name: qwen3-coder-next
  namespace: dynamo
  resourceVersion: "1557142663"
  uid: 72fd4290-804c-4176-80ed-d38e37ab1c70
spec:
  annotations:
    prometheus.io/probe_skip: "true"
  components:
  - compilationCache:
      mountPath: /home/dynamo/.cache/vllm
      pvcName: compilation-cache
    multinode:
      nodeCount: 8
    name: worker
    podTemplate:
      spec:
        containers:
        - args:
          - --model
          - Qwen/Qwen3-Coder-Next
          - --served-model-name
          - qwen3-coder-next
          - --tensor-parallel-size
          - "8"
          - --safetensors-load-strategy
          - eager
          - --max-model-len
          - "256000"
          - --dyn-tool-call-parser
          - qwen3_coder
          - --dyn-reasoning-parser
          - qwen3
          - --max-num-seqs
          - "512"
          - --max-num-batched-tokens
          - "8192"
          - --gpu-memory-utilization
          - "0.95"
          - --enable-chunked-prefill
          - --enable-prefix-caching
          command:
          - python3
          - -m
          - dynamo.vllm
          env:
          - name: VLLM_NO_USAGE_STATS
            value: "1"
          - name: DO_NOT_TRACK
            value: "1"
          - name: NIXL_TELEMETRY_ENABLE
            value: "y"
          - name: OMP_NUM_THREADS
            value: "2"
          envFrom:
          - secretRef:
              name: hf-token-secret
          image: docker-registry.osc.edu/kubernetes/ai-dynamo/vllm-runtime:1.2.0-cuda13-osc-r1
          imagePullPolicy: IfNotPresent
          name: main
          resources:
            limits:
              cpu: "8"
              memory: 32Gi
              nvidia.com/mig-7g.40gb: "1"
              rdma/shared_mlx5: "1"
            requests:
              cpu: "8"
              memory: 32Gi
              nvidia.com/mig-7g.40gb: "1"
              rdma/shared_mlx5: "1"
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            privileged: false
            seccompProfile:
              type: RuntimeDefault
          volumeMounts:
          - mountPath: /home/dynamo/.cache/huggingface
            name: model-cache
          workingDir: /home/dynamo
        initContainers:
        - args:
          - |
            set -x
            if kubectl get job qwen3-coder-next-download -n dynamo &>/dev/null; then
              echo "Waiting for download job to finish..."
              kubectl wait --for=condition=complete job/qwen3-coder-next-download -n dynamo --timeout=300s
              exit $?
            else
              echo "Job not found, exit"
              exit 0
            fi
          command:
          - /bin/bash
          - -c
          image: docker-registry.osc.edu/kubernetes/portainer/kubectl-shell:2.39.0
          imagePullPolicy: IfNotPresent
          name: hf-wait
          resources:
            limits:
              cpu: 500m
              memory: 128Mi
            requests:
              cpu: 250m
              memory: 64Mi
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            privileged: false
            seccompProfile:
              type: RuntimeDefault
        priorityClassName: inference
        volumes:
        - name: model-cache
          persistentVolumeClaim:
            claimName: model-cache
    replicas: 1
    scalingAdapter: {}
    type: worker
status:
  components:
    worker:
      availableReplicas: 1
      componentKind: PodCliqueScalingGroup
      componentNames:
      - qwen3-coder-next-0-worker
      replicas: 1
      updatedReplicas: 1
  conditions:
  - lastTransitionTime: "2026-07-17T16:11:07Z"
    message: All resources are ready
    reason: all_resources_are_ready
    status: "True"
    type: Ready
  observedGeneration: 3
  state: successful

helm values

    qwen3-coder-next:
      model: Qwen/Qwen3-Coder-Next
      alias: qwen3-coder-next
      disagg: false
      kvRouting: false
      scaling:
        minReplicas: 0
        maxReplicas: 2
      args:
        - --max-model-len
        - "256000"
        - --dyn-tool-call-parser
        - qwen3_coder
        - --dyn-reasoning-parser
        - qwen3
        - "--max-num-seqs"
        - '512'
        - --max-num-batched-tokens
        - '8192'
        - --gpu-memory-utilization
        - '0.95'
        - --enable-chunked-prefill
        - --enable-prefix-caching
      decode:
        cpu: 8
        memory: 32Gi
        gpu: 1
        nodes: 8
      prefill:
        cpu: 8
        memory: 32Gi
        gpu: 1
        nodes: 8

aiperf:

                                                  NVIDIA AIPerf | LLM Metrics
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━┓
┃                                        Metric ┃       avg ┃      min ┃       max ┃       p99 ┃       p90 ┃      p50 ┃    std ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━┩
│                      Time to First Token (ms) │  2,059.52 │   418.40 │  4,599.96 │  4,535.37 │  2,404.81 │ 2,161.20 │ 982.79 │
│                     Time to Second Token (ms) │    243.87 │     0.36 │  1,588.13 │    896.93 │    788.45 │   148.69 │ 281.78 │
│               Time to First Output Token (ms) │  2,059.52 │   418.40 │  4,599.96 │  4,535.37 │  2,404.81 │ 2,161.20 │ 982.79 │
│                          Request Latency (ms) │ 10,090.45 │ 9,770.10 │ 12,118.69 │ 12,117.55 │ 10,274.12 │ 9,882.84 │ 667.60 │
│                      Inter Token Latency (ms) │     31.50 │    29.25 │     39.50 │     37.14 │     36.22 │    30.27 │   2.58 │
│              Output Token Throughput Per User │     31.94 │    25.32 │     34.19 │     34.17 │     33.78 │    33.03 │   2.37 │
│                             (tokens/sec/user) │           │          │           │           │           │          │        │
│ E2E Output Token Throughput (tokens/sec/user) │     25.46 │    21.12 │     26.20 │     26.19 │     26.14 │    25.91 │   1.44 │
│               Output Sequence Length (tokens) │    255.99 │   255.00 │    257.00 │    256.01 │    256.00 │   256.00 │   0.17 │
│                Input Sequence Length (tokens) │  2,000.00 │ 2,000.00 │  2,000.00 │  2,000.00 │  2,000.00 │ 2,000.00 │   0.00 │
│          Output Token Throughput (tokens/sec) │    253.43 │      N/A │       N/A │       N/A │       N/A │      N/A │    N/A │
│             Request Throughput (requests/sec) │      0.99 │      N/A │       N/A │       N/A │       N/A │      N/A │    N/A │
│                      Request Count (requests) │    100.00 │      N/A │       N/A │       N/A │       N/A │      N/A │    N/A │
└───────────────────────────────────────────────┴───────────┴──────────┴───────────┴───────────┴───────────┴──────────┴────────┘


No GPU telemetry data collected during the benchmarking run.

CLI Command: aiperf profile --model 'qwen3-coder-next' --url 'https://ai-alpha.osc.edu/api' --endpoint-type 'chat' --streaming --concurrency 10
--request-count 100 --synthetic-input-tokens-mean 2000 --output-tokens-mean 256 --api-key '<redacted>' --tokenizer 'Qwen/Qwen3-Coder-Next'
--no-server-metrics
Benchmark Duration: 101.01 sec