mamba_repro.py

from transformers import AutoTokenizer, AutoModelForCausalLM
from optimum.intel.openvino import OVModelForCausalLM, OVWeightQuantizationConfig
import whowhatbench

model_dir = "state-spaces/mamba-130m-hf"
tokenizer = AutoTokenizer.from_pretrained(model_dir)

ov_model = AutoModelForCausalLM.from_pretrained(model_dir)

quantization_config = OVWeightQuantizationConfig(bits=4)
optimized_model = OVModelForCausalLM.from_pretrained(model_dir,quantization_config=quantization_config)
evaluator = whowhatbench.TextEvaluator(base_model=ov_model, tokenizer=tokenizer)
metrics_per_prompt, metrics = evaluator.score(optimized_model)

metric_of_interest = "similarity"
print(metric_of_interest, ": ", metrics["similarity"][0])

worst_examples = evaluator.worst_examples(top_k=5, metric=metric_of_interest)
print("Metric: ", metric_of_interest)
for e in worst_examples:
    print("\t=========================")
    print("\tPrompt: ", e["prompt"])
    print("\tBaseline Model:\n ", "\t" + e["source_model"])
    print("\tOptimized Model:\n ", "\t" + e["optimized_model"])

pip freeze 
about-time==4.2.1
accelerate==1.10.1
aiohappyeyeballs==2.6.1
aiohttp==3.12.15
aiosignal==1.4.0
alive-progress==3.3.0
async-timeout==5.0.1
attrs==25.3.0
auto_gptq==0.7.1
autoawq==0.2.7.post3
autograd==1.8.0
certifi==2025.8.3
charset-normalizer==3.4.3
cma==4.3.0
contourpy==1.3.2
cycler==0.12.1
datasets==3.6.0
Deprecated==1.2.18
diffusers==0.35.1
dill==0.3.8
filelock==3.19.1
fonttools==4.60.0
frozenlist==1.7.0
fsspec==2025.3.0
gekko==1.3.0
graphemeu==0.7.2
hf-xet==1.1.10
huggingface-hub==0.35.0
idna==3.10
importlib_metadata==8.7.0
Jinja2==3.1.6
joblib==1.5.2
jsonschema==4.25.1
jsonschema-specifications==2025.9.1
kiwisolver==1.4.9
markdown-it-py==4.0.0
MarkupSafe==3.0.2
matplotlib==3.10.6
mdurl==0.1.2
ml_dtypes==0.5.3
mpmath==1.3.0
multidict==6.6.4
multiprocess==0.70.16
natsort==8.4.0
netron==8.6.4
networkx==3.4.2
ninja==1.13.0
nncf==2.18.0
numpy==2.2.6
nvidia-cublas-cu12==12.8.4.1
nvidia-cuda-cupti-cu12==12.8.90
nvidia-cuda-nvrtc-cu12==12.8.93
nvidia-cuda-runtime-cu12==12.8.90
nvidia-cudnn-cu12==9.10.2.21
nvidia-cufft-cu12==11.3.3.83
nvidia-cufile-cu12==1.13.1.3
nvidia-curand-cu12==10.3.9.90
nvidia-cusolver-cu12==11.7.3.90
nvidia-cusparse-cu12==12.5.8.93
nvidia-cusparselt-cu12==0.7.1
nvidia-nccl-cu12==2.27.3
nvidia-nvjitlink-cu12==12.8.93
nvidia-nvtx-cu12==12.8.90
onnx==1.19.0
openvino==2025.3.0
openvino-genai==2025.3.0.0
openvino-telemetry==2025.2.0
openvino-tokenizers==2025.3.0.0
optimum==1.27.0
optimum-intel @ file:///home/dlyakhov/Projects/optimum-intel
packaging==25.0
pandas==2.3.2
peft==0.17.1
pillow==11.3.0
propcache==0.3.2
protobuf==6.32.1
psutil==7.0.0
pyarrow==21.0.0
pydot==3.0.4
Pygments==2.19.2
pymoo==0.6.1.5
pyparsing==3.2.4
python-dateutil==2.9.0.post0
pytz==2025.2
PyYAML==6.0.2
referencing==0.36.2
regex==2025.9.1
requests==2.32.5
rich==14.1.0
rouge==1.0.1
rpds-py==0.27.1
safetensors==0.6.2
scikit-learn==1.7.2
scipy==1.15.3
sentence-transformers==5.1.1
sentencepiece==0.2.1
six==1.17.0
sympy==1.14.0
tabulate==0.9.0
threadpoolctl==3.6.0
tokenizers==0.21.4
torch==2.8.0
tqdm==4.67.1
transformers==4.53.3
triton==3.4.0
typing_extensions==4.15.0
tzdata==2025.2
urllib3==2.5.0
whowhatbench @ file:///home/dlyakhov/Projects/openvino.genai/tools/who_what_benchmark
wrapt==1.17.3
xxhash==3.5.0
yarl==1.20.1
zipp==3.23.0
zstandard==0.25.0