[root@bastion yamls]# oc get pod
NAME READY STATUS RESTARTS AGE
gemma-2-9b-it-fp8-predictor-75fb9658bb-725mb 1/2 Running 0 15s
[root@bastion yamls]# oc logs -f gemma-2-9b-it-fp8-predictor-75fb9658bb-725mb
Defaulted container "kserve-container" out of: kserve-container, kube-rbac-proxy
[preflight] tcmalloc: /usr/lib64/libtcmalloc_minimal.so.4
[preflight] cpu_limit=8 world=1 (tp=1 pp=1 dp=1) api_servers=1 reserved=1
[preflight] OMP_NUM_THREADS=7 (per worker process, total <= 7x1+1)
[preflight] GPU compute capability: 8.9
[preflight] kernel coverage: sm_89 via family cubin (sm_8x series)
[preflight] WARNING: flashinfer-jit-cache is not installed.
[preflight] Every kernel will be compiled at runtime (minutes on first request),
[preflight] and it fails outright if the JIT toolchain is absent.
[preflight] Rebuild so the prebuilt kernel wheel is installed, or set
[preflight] VLLM_ATTENTION_BACKEND=TRITON_ATTN to avoid FlashInfer.
[preflight] JIT fallback : available (slow, but covers anything not prebuilt)
[preflight] /etc/passwd is read-only - skipped (HOME=/tmp is set, expected)
(APIServer pid=1) INFO 08-19 10:18:00 [api_utils.py:345]
(APIServer pid=1) INFO 08-19 10:18:00 [api_utils.py:345] █ █ █▄ ▄█
(APIServer pid=1) INFO 08-19 10:18:00 [api_utils.py:345] ▄▄ ▄█ █ █ █ ▀▄▀ █ version 0.27.1
(APIServer pid=1) INFO 08-19 10:18:00 [api_utils.py:345] █▄█▀ █ █ █ █ model /mnt/models
(APIServer pid=1) INFO 08-19 10:18:00 [api_utils.py:345] ▀▀ ▀▀▀▀▀ ▀▀▀▀▀ ▀ ▀
(APIServer pid=1) INFO 08-19 10:18:00 [api_utils.py:345]
(APIServer pid=1) INFO 08-19 10:18:00 [api_utils.py:273] non-default args: {'enable_prompt_tokens_details': True, 'port': 8080, 'enable_request_id_headers': True, 'disable_fastapi_docs': True, 'model': '/mnt/models', 'max_model_len': 8192, 'served_model_name': ['gemma-2-9b-it-fp8'], 'distributed_timeout_seconds': 3600, 'gpu_memory_utilization': 0.9, 'kv_cache_dtype': 'fp8', 'enable_prefix_caching': True, 'kv_cache_metrics': True, 'enable_mfu_metrics': True, 'max_num_batched_tokens': 2048, 'max_num_seqs': 8, 'long_prefill_token_threshold': 1024, 'enable_chunked_prefill': True, 'async_scheduling': True, 'shutdown_timeout': 120}
(APIServer pid=1) INFO 08-19 10:18:07 [model.py:645] Resolved architecture: Gemma2ForCausalLM
(APIServer pid=1) INFO 08-19 10:18:07 [model.py:1883] Using max model len 8192
(APIServer pid=1) INFO 08-19 10:18:07 [cache.py:296] Using fp8 data type to store kv cache. It reduces the GPU memory footprint and boosts the performance. Meanwhile, it may cause accuracy drop without a proper scaling factor
(APIServer pid=1) INFO 08-19 10:18:07 [kernel.py:306] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
(EngineCore pid=150) INFO 08-19 10:18:19 [core.py:121] Initializing a V1 LLM engine (v0.27.1) with config: model='/mnt/models', speculative_config=None, tokenizer='/mnt/models', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=8192, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=fp8, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=fp8, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=True, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=True, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=gemma-2-9b-it-fp8, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::qwen_gdn_attention_core', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::hpc_rope_norm_forward', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 16, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
(EngineCore pid=150) INFO 08-19 10:18:19 [parallel_state.py:1640] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://10.131.1.253:43653 backend=nccl
(EngineCore pid=150) INFO 08-19 10:18:19 [parallel_state.py:1977] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
(EngineCore pid=150) INFO 08-19 10:18:19 [gpu_worker.py:385] Using V2 Model Runner
(EngineCore pid=150) INFO 08-19 10:18:20 [model_runner.py:308] Loading model from scratch...
(EngineCore pid=150) INFO 08-19 10:18:20 [__init__.py:665] Selected CutlassFP8ScaledMMLinearKernel for Fp8LinearMethod
(EngineCore pid=150) INFO 08-19 10:18:20 [cuda.py:482] Using FLASHINFER attention backend out of potential backends: ['FLASHINFER', 'TRITON_ATTN'].
(EngineCore pid=150) INFO 08-19 10:18:21 [weight_utils.py:867] Filesystem type for checkpoints: NFS4. Checkpoint size: 9.46 GiB. Available RAM: 13.73 GiB.
(EngineCore pid=150) INFO 08-19 10:18:21 [weight_utils.py:829] Prefetching checkpoint files into page cache started (in background, num_threads=8, block_size=16777216 bytes)
Loading safetensors checkpoint shards: 0% Completed | 0/3 [00:00<?, ?it/s]
(EngineCore pid=150) INFO 08-19 10:18:21 [weight_utils.py:801] Prefetching checkpoint files: 10% (1/3)
(EngineCore pid=150) INFO 08-19 10:18:23 [weight_utils.py:801] Prefetching checkpoint files: 20% (2/3)
(EngineCore pid=150) INFO 08-19 10:18:26 [weight_utils.py:801] Prefetching checkpoint files: 30% (3/3)
(EngineCore pid=150) INFO 08-19 10:18:26 [weight_utils.py:824] Prefetching checkpoint files into page cache finished in 5.00s
Loading safetensors checkpoint shards: 33% Completed | 1/3 [00:04<00:09, 5.00s/it]
Loading safetensors checkpoint shards: 67% Completed | 2/3 [00:05<00:02, 2.50s/it]
Loading safetensors checkpoint shards: 100% Completed | 3/3 [00:05<00:00, 1.41s/it]
Loading safetensors checkpoint shards: 100% Completed | 3/3 [00:05<00:00, 1.96s/it]
(EngineCore pid=150)
(EngineCore pid=150) INFO 08-19 10:18:27 [default_loader.py:430] Loading weights took 5.87 seconds
(EngineCore pid=150) WARNING 08-19 10:18:27 [kv_cache.py:134] Checkpoint does not provide a q scaling factor. Setting it to k_scale. This only matters for FP8 Attention backends (flash-attn or flashinfer).
(EngineCore pid=150) WARNING 08-19 10:18:27 [kv_cache.py:151] Using KV cache scaling factor 1.0 for fp8_e4m3. If this is unintended, verify that k/v_scale scaling factors are properly set in the checkpoint.
(EngineCore pid=150) WARNING 08-19 10:18:27 [kv_cache.py:190] Using uncalibrated q_scale 1.0 and/or prob_scale 1.0 with fp8 attention. This may cause accuracy issues. Please make sure q/prob scaling factors are available in the fp8 checkpoint.
(EngineCore pid=150) INFO 08-19 10:18:28 [model_runner.py:329] Model loading took 9.47 GiB and 8.572459 seconds
(EngineCore pid=150) INFO 08-19 10:18:28 [topk_topp_sampler.py:62] Using FlashInfer for top-p & top-k sampling.
(EngineCore pid=150) INFO 08-19 10:18:40 [backends.py:1094] Using cache directory: /tmp/vllm/torch_compile_cache/9cfaa2212f/rank_0_0/backbone for vLLM's torch.compile
(EngineCore pid=150) INFO 08-19 10:18:40 [backends.py:1155] Dynamo bytecode transform time: 10.96 s
(EngineCore pid=150) INFO 08-19 10:18:48 [backends.py:393] Compiling a graph for compile range (1, 2048) takes 7.24 s
(EngineCore pid=150) INFO 08-19 10:18:54 [backends.py:920] collected artifacts: 43 entries, 3 artifacts, 6379997 bytes total
(EngineCore pid=150) INFO 08-19 10:18:54 [decorators.py:708] saved AOT compiled function to /tmp/vllm/torch_compile_cache/torch_aot_compile/c745aa1b4a9ee5381cb452f0657c80c023e41c0396e905014dd85ebef33025e5/rank_0_0/model
(EngineCore pid=150) INFO 08-19 10:18:54 [monitor.py:53] torch.compile took 25.47 s in total
(EngineCore pid=150) INFO 08-19 10:18:55 [monitor.py:81] Initial profiling/warmup run took 0.24 s
(EngineCore pid=150) INFO 08-19 10:18:56 [gpu_worker.py:563] Available KV cache memory: 8.45 GiB
(EngineCore pid=150) INFO 08-19 10:18:56 [kv_cache_utils.py:2235] GPU KV cache size: 52,676 tokens
(EngineCore pid=150) INFO 08-19 10:18:56 [kv_cache_utils.py:2236] Maximum concurrency for 8,192 tokens per request: 6.43x
(EngineCore pid=150) INFO 08-19 10:18:56 [flashinfer.py:824] FlashInfer resolved query dtypes: prefill=torch.bfloat16, decode=torch.bfloat16, decode_backend=flashinfer-native, kv_cache_dtype=torch.float8_e4m3fn, arch=sm89
(EngineCore pid=150) INFO 08-19 10:18:57 [kernel_warmup.py:191] Warming up FlashInfer attention.
Capturing CUDA graphs (PIECEWISE): 100%|██████████| 5/5 [00:00<00:00, 15.08it/s]
Capturing CUDA graphs (FULL): 100%|██████████| 4/4 [00:00<00:00, 19.63it/s]
(EngineCore pid=150) INFO 08-19 10:19:52 [model_runner.py:791] Graph capturing finished in 1 secs, took 0.13 GiB
(EngineCore pid=150) INFO 08-19 10:19:52 [gpu_worker.py:789] Free memory on device (21.84/22.03 GiB) on startup. Desired GPU memory utilization is (0.9, 19.83 GiB). Actual usage is 9.65 GiB for consumed memory (weights + non-torch), 1.74 GiB for peak activation, and 0.13 GiB for CUDAGraph memory. Replace gpu_memory_utilization config with `--kv-cache-memory=8774400000` (8.17 GiB) to fit into requested memory, or `--kv-cache-memory=10936891392` (10.19 GiB) to fully utilize gpu memory. Current kv cache memory in use is 8.45 GiB.
(EngineCore pid=150) INFO 08-19 10:20:48 [jit_monitor.py:79] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn.
(EngineCore pid=150) INFO 08-19 10:20:49 [core.py:348] init engine (profile, create kv cache, warmup model) took 140.88 s (compilation: 25.47 s)
(EngineCore pid=150) INFO 08-19 10:20:50 [perf.py:1282] Instantiated ComponentMetrics [attn] with (num_hidden_layers=42 hidden_size=3584 num_attention_heads=16 activation_byte_size=2 tp_size=1 pp_size=1 num_key_value_heads=8 head_dim=256 cache_byte_size=1 weight_byte_size=1)
(EngineCore pid=150) INFO 08-19 10:20:50 [perf.py:1282] Instantiated ComponentMetrics [ffn] with (num_hidden_layers=42 hidden_size=3584 activation_byte_size=2 pp_size=1 ffn_tp_size=1 ffn_ep_size=1 intermediate_size=14336 num_experts=0 num_experts_per_tok=0 moe_intermediate_size=14336 num_shared_experts=0 num_moe_layers=0 weight_byte_size=1)
(EngineCore pid=150) INFO 08-19 10:20:50 [perf.py:1282] Instantiated ComponentMetrics [unembed] with (hidden_size=3584 vocab_size=256000 weight_byte_size=2 activation_byte_size=2 tp_size=1)
(EngineCore pid=150) INFO 08-19 10:20:50 [kernel.py:306] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
(APIServer pid=1) INFO 08-19 10:20:50 [api_server.py:678] Supported tasks: ['generate']
(APIServer pid=1) INFO 08-19 10:20:51 [hf.py:540] Detected the chat template content format to be 'string'. You can set `--chat-template-content-format` to override this.
(APIServer pid=1) INFO 08-19 10:20:51 [api_server.py:682] Starting vLLM server on http://0.0.0.0:8080
(APIServer pid=1) INFO 08-19 10:20:51 [launcher.py:37] Available routes are:
(APIServer pid=1) INFO 08-19 10:20:51 [launcher.py:46] Route: /load, Methods: GET
(APIServer pid=1) INFO 08-19 10:20:51 [launcher.py:46] Route: /version, Methods: GET
(APIServer pid=1) INFO 08-19 10:20:51 [launcher.py:46] Route: /health, Methods: GET
(APIServer pid=1) INFO 08-19 10:20:51 [launcher.py:46] Route: /metrics, Methods: GET
(APIServer pid=1) INFO 08-19 10:20:51 [launcher.py:46] Route: /tokenize, Methods: POST
(APIServer pid=1) INFO 08-19 10:20:51 [launcher.py:46] Route: /detokenize, Methods: POST
(APIServer pid=1) INFO 08-19 10:20:51 [launcher.py:46] Route: /v1/models, Methods: GET
(APIServer pid=1) INFO 08-19 10:20:51 [launcher.py:46] Route: /ping, Methods: GET
(APIServer pid=1) INFO 08-19 10:20:51 [launcher.py:46] Route: /ping, Methods: POST
(APIServer pid=1) INFO 08-19 10:20:51 [launcher.py:46] Route: /invocations, Methods: POST
(APIServer pid=1) INFO 08-19 10:20:51 [launcher.py:46] Route: /v1/chat/completions, Methods: POST
(APIServer pid=1) INFO 08-19 10:20:51 [launcher.py:46] Route: /v1/chat/completions/batch, Methods: POST
(APIServer pid=1) INFO 08-19 10:20:51 [launcher.py:46] Route: /v1/responses, Methods: POST
(APIServer pid=1) INFO 08-19 10:20:51 [launcher.py:46] Route: /v1/responses/{response_id}, Methods: GET
(APIServer pid=1) INFO 08-19 10:20:51 [launcher.py:46] Route: /v1/responses/{response_id}/cancel, Methods: POST
(APIServer pid=1) INFO 08-19 10:20:51 [launcher.py:46] Route: /v1/completions, Methods: POST
(APIServer pid=1) INFO 08-19 10:20:51 [launcher.py:46] Route: /v1/messages, Methods: POST
(APIServer pid=1) INFO 08-19 10:20:51 [launcher.py:46] Route: /v1/messages/count_tokens, Methods: POST
(APIServer pid=1) INFO 08-19 10:20:51 [launcher.py:46] Route: /generative_scoring, Methods: POST
(APIServer pid=1) INFO 08-19 10:20:51 [launcher.py:46] Route: /scale_elastic_ep, Methods: POST
(APIServer pid=1) INFO 08-19 10:20:51 [launcher.py:46] Route: /is_scaling_elastic_ep, Methods: POST
(APIServer pid=1) INFO 08-19 10:20:51 [launcher.py:46] Route: /v1/chat/completions/render, Methods: POST
(APIServer pid=1) INFO 08-19 10:20:51 [launcher.py:46] Route: /v1/completions/render, Methods: POST
(APIServer pid=1) INFO 08-19 10:20:51 [launcher.py:46] Route: /v1/chat/completions/derender, Methods: POST
(APIServer pid=1) INFO 08-19 10:20:51 [launcher.py:46] Route: /v1/completions/derender, Methods: POST
(APIServer pid=1) INFO 08-19 10:20:51 [launcher.py:46] Route: /inference/v1/generate, Methods: POST
(APIServer pid=1) INFO 08-19 10:20:51 [launcher.py:99] API server: waiting for HTTP server to start
(APIServer pid=1) INFO: Started server process [1]
(APIServer pid=1) INFO: Waiting for application startup.
(APIServer pid=1) INFO: Application startup complete.
(APIServer pid=1) INFO 08-19 10:20:51 [launcher.py:105] API server: HTTP server started
(APIServer pid=1) INFO: 10.131.0.2:36828 - "GET /health HTTP/1.1" 200 OK
(APIServer pid=1) INFO: 10.131.0.2:36838 - "GET /health HTTP/1.1" 200 OK
(APIServer pid=1) INFO: 10.131.0.2:35614 - "GET /health HTTP/1.1" 200 OK
(APIServer pid=1) INFO: 10.131.0.2:40010 - "GET /health HTTP/1.1" 200 OK
(APIServer pid=1) INFO: 10.131.0.2:33586 - "GET /health HTTP/1.1" 200 OK
(APIServer pid=1) INFO: 10.131.0.2:33592 - "GET /health HTTP/1.1" 200 OK
(APIServer pid=1) INFO: 10.131.0.2:37116 - "GET /health HTTP/1.1" 200 OK
(APIServer pid=1) INFO: 10.131.0.2:40852 - "GET /health HTTP/1.1" 200 OK
(APIServer pid=1) INFO: 10.131.0.2:33146 - "GET /health HTTP/1.1" 200 OK
(APIServer pid=1) INFO: 10.131.0.2:33162 - "GET /health HTTP/1.1" 200 OK
(APIServer pid=1) INFO: 10.131.0.2:44658 - "GET /health HTTP/1.1" 200 OK
