Skip to content

Instantly share code, notes, and snippets.

@daniil-lyakhov

daniil-lyakhov/mamba_repro.py

Created September 26, 2025 12:18
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save daniil-lyakhov/3f2c91496d343a12cfa007f3694d4002 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save daniil-lyakhov/3f2c91496d343a12cfa007f3694d4002 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
mamba_repro.py
from transformers import AutoTokenizer, AutoModelForCausalLM
from optimum.intel.openvino import OVModelForCausalLM, OVWeightQuantizationConfig
import whowhatbench
model_dir = "state-spaces/mamba-130m-hf"
tokenizer = AutoTokenizer.from_pretrained(model_dir)
ov_model = AutoModelForCausalLM.from_pretrained(model_dir)
quantization_config = OVWeightQuantizationConfig(bits=4)
optimized_model = OVModelForCausalLM.from_pretrained(model_dir,quantization_config=quantization_config)
evaluator = whowhatbench.TextEvaluator(base_model=ov_model, tokenizer=tokenizer)
metrics_per_prompt, metrics = evaluator.score(optimized_model)
metric_of_interest = "similarity"
print(metric_of_interest, ": ", metrics["similarity"][0])
worst_examples = evaluator.worst_examples(top_k=5, metric=metric_of_interest)
print("Metric: ", metric_of_interest)
for e in worst_examples:
print("\t=========================")
print("\tPrompt: ", e["prompt"])
print("\tBaseline Model:\n ", "\t" + e["source_model"])
print("\tOptimized Model:\n ", "\t" + e["optimized_model"])
@daniil-lyakhov
Copy link
Author

daniil-lyakhov commented Sep 26, 2025 

Traceback (most recent call last):
  File "/home/dlyakhov/Projects/optimum-intel/mamba_repro.py", line 14, in <module>
    metrics_per_prompt, metrics = evaluator.score(optimized_model)
  File "/home/dlyakhov/Projects/optimum-intel/env/lib/python3.10/site-packages/whowhatbench/text_evaluator.py", line 93, in score
    predictions = self._generate_data(model_or_data, gen_answer_fn, self.generation_config)
  File "/home/dlyakhov/Projects/optimum-intel/env/lib/python3.10/site-packages/whowhatbench/text_evaluator.py", line 186, in _generate_data
    gen_answer_fn(
  File "/home/dlyakhov/Projects/optimum-intel/env/lib/python3.10/site-packages/whowhatbench/text_evaluator.py", line 151, in default_gen_answer
    tokens = model.generate(**inputs, do_sample=False, max_new_tokens=max_new_tokens, **get_ignore_parameters_flag())
  File "/home/dlyakhov/Projects/optimum-intel/env/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 120, in decorate_context
    return func(*args, **kwargs)
  File "/home/dlyakhov/Projects/optimum-intel/optimum/intel/openvino/modeling_decoder.py", line 764, in generate
    result = super().generate(
  File "/home/dlyakhov/Projects/optimum-intel/env/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 120, in decorate_context
    return func(*args, **kwargs)
  File "/home/dlyakhov/Projects/optimum-intel/env/lib/python3.10/site-packages/transformers/generation/utils.py", line 2625, in generate
    result = self._sample(
  File "/home/dlyakhov/Projects/optimum-intel/env/lib/python3.10/site-packages/transformers/generation/utils.py", line 3606, in _sample
    outputs = self(**model_inputs, return_dict=True)
  File "/home/dlyakhov/Projects/optimum-intel/env/lib/python3.10/site-packages/optimum/modeling_base.py", line 111, in __call__
    return self.forward(*args, **kwargs)
  File "/home/dlyakhov/Projects/optimum-intel/optimum/intel/openvino/modeling_decoder.py", line 1227, in forward
    self.request.start_async(inputs, share_inputs=True)
AttributeError: 'NoneType' object has no attribute 'start_async'

@daniil-lyakhov
Copy link
Author

daniil-lyakhov commented Sep 26, 2025 

pip freeze 
about-time==4.2.1
accelerate==1.10.1
aiohappyeyeballs==2.6.1
aiohttp==3.12.15
aiosignal==1.4.0
alive-progress==3.3.0
async-timeout==5.0.1
attrs==25.3.0
auto_gptq==0.7.1
autoawq==0.2.7.post3
autograd==1.8.0
certifi==2025.8.3
charset-normalizer==3.4.3
cma==4.3.0
contourpy==1.3.2
cycler==0.12.1
datasets==3.6.0
Deprecated==1.2.18
diffusers==0.35.1
dill==0.3.8
filelock==3.19.1
fonttools==4.60.0
frozenlist==1.7.0
fsspec==2025.3.0
gekko==1.3.0
graphemeu==0.7.2
hf-xet==1.1.10
huggingface-hub==0.35.0
idna==3.10
importlib_metadata==8.7.0
Jinja2==3.1.6
joblib==1.5.2
jsonschema==4.25.1
jsonschema-specifications==2025.9.1
kiwisolver==1.4.9
markdown-it-py==4.0.0
MarkupSafe==3.0.2
matplotlib==3.10.6
mdurl==0.1.2
ml_dtypes==0.5.3
mpmath==1.3.0
multidict==6.6.4
multiprocess==0.70.16
natsort==8.4.0
netron==8.6.4
networkx==3.4.2
ninja==1.13.0
nncf==2.18.0
numpy==2.2.6
nvidia-cublas-cu12==12.8.4.1
nvidia-cuda-cupti-cu12==12.8.90
nvidia-cuda-nvrtc-cu12==12.8.93
nvidia-cuda-runtime-cu12==12.8.90
nvidia-cudnn-cu12==9.10.2.21
nvidia-cufft-cu12==11.3.3.83
nvidia-cufile-cu12==1.13.1.3
nvidia-curand-cu12==10.3.9.90
nvidia-cusolver-cu12==11.7.3.90
nvidia-cusparse-cu12==12.5.8.93
nvidia-cusparselt-cu12==0.7.1
nvidia-nccl-cu12==2.27.3
nvidia-nvjitlink-cu12==12.8.93
nvidia-nvtx-cu12==12.8.90
onnx==1.19.0
openvino==2025.3.0
openvino-genai==2025.3.0.0
openvino-telemetry==2025.2.0
openvino-tokenizers==2025.3.0.0
optimum==1.27.0
optimum-intel @ file:///home/dlyakhov/Projects/optimum-intel
packaging==25.0
pandas==2.3.2
peft==0.17.1
pillow==11.3.0
propcache==0.3.2
protobuf==6.32.1
psutil==7.0.0
pyarrow==21.0.0
pydot==3.0.4
Pygments==2.19.2
pymoo==0.6.1.5
pyparsing==3.2.4
python-dateutil==2.9.0.post0
pytz==2025.2
PyYAML==6.0.2
referencing==0.36.2
regex==2025.9.1
requests==2.32.5
rich==14.1.0
rouge==1.0.1
rpds-py==0.27.1
safetensors==0.6.2
scikit-learn==1.7.2
scipy==1.15.3
sentence-transformers==5.1.1
sentencepiece==0.2.1
six==1.17.0
sympy==1.14.0
tabulate==0.9.0
threadpoolctl==3.6.0
tokenizers==0.21.4
torch==2.8.0
tqdm==4.67.1
transformers==4.53.3
triton==3.4.0
typing_extensions==4.15.0
tzdata==2025.2
urllib3==2.5.0
whowhatbench @ file:///home/dlyakhov/Projects/openvino.genai/tools/who_what_benchmark
wrapt==1.17.3
xxhash==3.5.0
yarl==1.20.1
zipp==3.23.0
zstandard==0.25.0

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment