RTX 4070 12GB VRAM, 64GB RAM
git fetch origin pull/23398/head:gemma-mtp
git checkout gemma-mtpllama.cpp/build/bin/llama-server -m /path-to-models/unsloth/gemma-4-31B-it-UD-Q5_K_XL.gguf --ctx-size 131072 --flash-attn on --no-mmap --fit on --temp 1 --top-p 0.95 --top-k 64 --host 0.0.0.0 --port 8381 --cache-type-k q8_0 --cache-type-v q8_0| Test | Speed |
|---|---|
| code_python | pred= 192 draft= 0 acc= 0 rate=n/a tok/s=60.0 |
| code_cpp | pred= 192 draft= 0 acc= 0 rate=n/a tok/s=60.1 |
| explain_concept | pred= 192 draft= 0 acc= 0 rate=n/a tok/s=60.1 |
| summarize | pred= 192 draft= 0 acc= 0 rate=n/a tok/s=60.0 |
| qa_factual | pred= 192 draft= 0 acc= 0 rate=n/a tok/s=60.0 |
| translation | pred= 192 draft= 0 acc= 0 rate=n/a tok/s=60.0 |
| creative_short | pred= 192 draft= 0 acc= 0 rate=n/a tok/s=60.0 |
| stepwise_math | pred= 192 draft= 0 acc= 0 rate=n/a tok/s=59.9 |
| long_code_review | pred= 192 draft= 0 acc= 0 rate=n/a tok/s=59.2 |
Aggregate: { "n_requests": 9, "total_predicted": 1728, "total_draft": 0, "total_draft_accepted": 0, "aggregate_accept_rate": null, "wall_s_total": 29.92 }
{
"results": [
{
"name": "code_python",
"wall_s": 3.309,
"predicted_n": 192,
"predicted_per_second": 60.01,
"draft_n": 0,
"draft_n_accepted": 0,
"accept_rate": null
},
{
"name": "code_cpp",
"wall_s": 3.29,
"predicted_n": 192,
"predicted_per_second": 60.08,
"draft_n": 0,
"draft_n_accepted": 0,
"accept_rate": null
},
{
"name": "explain_concept",
"wall_s": 3.287,
"predicted_n": 192,
"predicted_per_second": 60.08,
"draft_n": 0,
"draft_n_accepted": 0,
"accept_rate": null
},
{
"name": "summarize",
"wall_s": 3.298,
"predicted_n": 192,
"predicted_per_second": 59.97,
"draft_n": 0,
"draft_n_accepted": 0,
"accept_rate": null
},
{
"name": "qa_factual",
"wall_s": 3.291,
"predicted_n": 192,
"predicted_per_second": 60.03,
"draft_n": 0,
"draft_n_accepted": 0,
"accept_rate": null
},
{
"name": "translation",
"wall_s": 3.292,
"predicted_n": 192,
"predicted_per_second": 59.97,
"draft_n": 0,
"draft_n_accepted": 0,
"accept_rate": null
},
{
"name": "creative_short",
"wall_s": 3.287,
"predicted_n": 192,
"predicted_per_second": 60.05,
"draft_n": 0,
"draft_n_accepted": 0,
"accept_rate": null
},
{
"name": "stepwise_math",
"wall_s": 3.299,
"predicted_n": 192,
"predicted_per_second": 59.94,
"draft_n": 0,
"draft_n_accepted": 0,
"accept_rate": null
},
{
"name": "long_code_review",
"wall_s": 3.569,
"predicted_n": 192,
"predicted_per_second": 59.18,
"draft_n": 0,
"draft_n_accepted": 0,
"accept_rate": null
}
],
"aggregate": {
"n_requests": 9,
"total_predicted": 1728,
"total_draft": 0,
"total_draft_accepted": 0,
"aggregate_accept_rate": null,
"wall_s_total": 29.92
}
}llama.cpp/build/bin/llama-server -m /path-to-models/unsloth/gemma-4-31B-it-UD-Q5_K_XL.gguf --ctx-size 131072 --flash-attn on --no-mmap --fit on --temp 1 --top-p 0.95 --top-k 64 --host 0.0.0.0 --port 8381 --cache-type-k q8_0 --cache-type-v q8_0 --spec-type draft-mtp --spec-draft-n-max 2 --spec-draft-model /path-to-models/am17an/mtp-gemma-4-31B-it.gguf| Test | Speed |
|---|---|
| code_python | pred= 192 draft= 157 acc= 112 rate=0.713 tok/s=96.2 |
| code_cpp | pred= 192 draft= 157 acc= 111 rate=0.707 tok/s=95.7 |
| explain_concept | pred= 192 draft= 161 acc= 109 rate=0.677 tok/s=93.2 |
| summarize | pred= 192 draft= 149 acc= 115 rate=0.772 tok/s=100.4 |
| qa_factual | pred= 192 draft= 164 acc= 109 rate=0.665 tok/s=91.2 |
| translation | pred= 192 draft= 150 acc= 115 rate=0.767 tok/s=99.8 |
| creative_short | pred= 192 draft= 188 acc= 96 rate=0.511 tok/s=79.1 |
| stepwise_math | pred= 192 draft= 145 acc= 118 rate=0.814 tok/s=99.8 |
| long_code_review | pred= 192 draft= 172 acc= 105 rate=0.611 tok/s=84.4 |
Aggregate: { "n_requests": 9, "total_predicted": 1728, "total_draft": 1443, "total_draft_accepted": 990, "aggregate_accept_rate": 0.6861, "wall_s_total": 20.06 }
{
"results": [
{
"name": "code_python",
"wall_s": 2.345,
"predicted_n": 192,
"predicted_per_second": 96.22,
"draft_n": 157,
"draft_n_accepted": 112,
"accept_rate": 0.7134
},
{
"name": "code_cpp",
"wall_s": 2.102,
"predicted_n": 192,
"predicted_per_second": 95.65,
"draft_n": 157,
"draft_n_accepted": 111,
"accept_rate": 0.707
},
{
"name": "explain_concept",
"wall_s": 2.154,
"predicted_n": 192,
"predicted_per_second": 93.15,
"draft_n": 161,
"draft_n_accepted": 109,
"accept_rate": 0.677
},
{
"name": "summarize",
"wall_s": 2.017,
"predicted_n": 192,
"predicted_per_second": 100.44,
"draft_n": 149,
"draft_n_accepted": 115,
"accept_rate": 0.7718
},
{
"name": "qa_factual",
"wall_s": 2.207,
"predicted_n": 192,
"predicted_per_second": 91.16,
"draft_n": 164,
"draft_n_accepted": 109,
"accept_rate": 0.6646
},
{
"name": "translation",
"wall_s": 2.021,
"predicted_n": 192,
"predicted_per_second": 99.84,
"draft_n": 150,
"draft_n_accepted": 115,
"accept_rate": 0.7667
},
{
"name": "creative_short",
"wall_s": 2.524,
"predicted_n": 192,
"predicted_per_second": 79.14,
"draft_n": 188,
"draft_n_accepted": 96,
"accept_rate": 0.5106
},
{
"name": "stepwise_math",
"wall_s": 2.03,
"predicted_n": 192,
"predicted_per_second": 99.75,
"draft_n": 145,
"draft_n_accepted": 118,
"accept_rate": 0.8138
},
{
"name": "long_code_review",
"wall_s": 2.663,
"predicted_n": 192,
"predicted_per_second": 84.43,
"draft_n": 172,
"draft_n_accepted": 105,
"accept_rate": 0.6105
}
],
"aggregate": {
"n_requests": 9,
"total_predicted": 1728,
"total_draft": 1443,
"total_draft_accepted": 990,
"aggregate_accept_rate": 0.6861,
"wall_s_total": 20.06
}
}