Simple python script to run for small LLMs benchmark on Intel Arc A40

ai-bench.py

import requests, time

def warmup(model, prompt):
    requests.post("http://ai:11434/api/generate",
        json={"model": model, "prompt": prompt, "stream": False, "keep_alive": -1})

def run(model, prompt):
    start = time.time()
    r = requests.post("http://ai:11434/api/generate",
        json={"model": model, "prompt": prompt, "stream": False, "keep_alive": -1})
    elapsed = time.time() - start

    data = r.json()
    print(f"  Gen time : {data['eval_duration'] / 1e9:.2f}s")
    print(f"  Tokens/s : {data['eval_count'] / elapsed:.1f}")
    print(f"  Tokens   : {data['eval_count']}")

def unload(model):
    requests.post("http://ai:11434/api/generate",
        json={"model": model, "prompt": "", "keep_alive": 0})

models = [
    "phi3:mini",
    "llama3.1:8b",
    "mistral:7b",
    "qwen2.5:7b-instruct-q4_K_M",
    "gemma3:4b",
]

prompts = {
    "short":  "What is the capital of Australia?",
    "reason": "If a train travels 120km in 1.5 hours, what is its average speed? Show your working.",
    "code":   "Write a Python function that checks if a string is a palindrome.",
    "long":   "Explain how transformers work in machine learning, in detail.",
}

for model in models:
    print(f"\n====== {model} ======")
    warmup(model, prompts["short"])

    for label, prompt in prompts.items():
        print(f"\n  -- {label} --")
        run(model, prompt)

    unload(model)