Benchmarks for mlx-lm
The command for evaluating on MMLU Pro:
mlx_lm.evaluate --model model/repo --task mmlu_pro
The command for efficiency benchmarks:
| Benchmark started on 2025-10-19 22:52:10 | |
| ** Command line: | |
| /Users/ifioravanti/github/consumer-tflop-database/.venv/bin/python mamf-finder.py --m_range 0 16384 1024 --n_range 0 16384 1024 --k_range 0 16384 1024 --dtype bfloat16 --output_file=2025-10-19-22:52:09.txt | |
| ** Dtype: torch.bfloat16 | |
| ** Platform/Device info: | |
| - Darwin MacStudioIvan 25.1.0 Darwin Kernel Version 25.1.0: Sun Oct 5 21:09:25 PDT 2025; root:xnu-12377.40.120~10/RELEASE_ARM64_T6031 arm64 arm |
The command for evaluating on MMLU Pro:
mlx_lm.evaluate --model model/repo --task mmlu_pro
The command for efficiency benchmarks:
| """Run four batched generations with varying sampling settings.""" | |
| import argparse | |
| import mlx.core as mx | |
| from mlx_lm import batch_generate, load | |
| from mlx_lm.sample_utils import make_sampler |
| #!/usr/bin/env python3 | |
| """ | |
| MLX benchmark script that replicates llama-bench behavior exactly. | |
| Uses random tokens for both prompt and generation, no sampling. | |
| """ | |
| import mlx.core as mx | |
| import mlx_lm | |
| from mlx_lm.models.cache import make_prompt_cache | |
| import time |
| import pygame | |
| import math | |
| import random | |
| # Initialize pygame | |
| pygame.init() | |
| # Screen dimensions | |
| WIDTH, HEIGHT = 800, 800 | |
| screen = pygame.display.set_mode((WIDTH, HEIGHT)) |
| <!DOCTYPE html> | |
| <html lang="en"> | |
| <head> | |
| <meta charset="UTF-8"> | |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> | |
| <title>P5.js Particle Animation</title> | |
| <script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/1.4.0/p5.min.js"></script> | |
| <style> | |
| body { | |
| margin: 0; |
| import pygame | |
| import math | |
| import sys | |
| pygame.init() | |
| # Screen setup | |
| WIDTH, HEIGHT = 800, 600 | |
| screen = pygame.display.set_mode((WIDTH, HEIGHT)) | |
| pygame.display.set_caption("Bouncing Ball in Rotating Square") |
| #!/usr/bin/env bash | |
| # Default values for percentages | |
| DEFAULT_WIRED_LIMIT_PERCENT=85 | |
| DEFAULT_WIRED_LWM_PERCENT=75 | |
| # Read input parameters or use default values | |
| WIRED_LIMIT_PERCENT=${1:-$DEFAULT_WIRED_LIMIT_PERCENT} | |
| WIRED_LWM_PERCENT=${2:-$DEFAULT_WIRED_LWM_PERCENT} |
| import pygame | |
| import random | |
| import math | |
| import os | |
| # Initialize Pygame | |
| pygame.init() | |
| # Screen dimensions | |
| WIDTH, HEIGHT = 800, 600 |
| import argparse | |
| import json | |
| import logging | |
| import os | |
| import re | |
| import shutil | |
| from concurrent.futures import ProcessPoolExecutor, as_completed | |
| from dataclasses import dataclass | |
| from datetime import datetime | |
| from typing import Any, Callable, Dict, List, Literal, Optional, Tuple |