vuiseng9

import intel_extension_for_pytorch # requried for XPU
import torch
from bigdl.llm.transformers import AutoModelForCausalLM
from transformers import AutoTokenizer, pipeline

# model_id = "facebook/opt-1.3b"
# model_id = "meta-llama/Llama-2-7b"
model_id = "meta-llama/Llama-2-7b-chat-hf"
prompt = "I love the Avengers,"

vuiseng9

$ python bench_linear.py --bs 1
BS:    1, Latency:    0.389 ms, IC:  4096, OC: 11008, Samples: 100, Warmup: 10

$ python bench_linear.py --bs 128
BS:  128, Latency:    3.640 ms, IC:  4096, OC: 11008, Samples: 100, Warmup: 10

$ python bench_linear.py --bs 1024
BS: 1024, Latency:   41.244 ms, IC:  4096, OC: 11008, Samples: 100, Warmup: 10

vuiseng9

Install

pip install transformers torch accelerate

Run

from transformers import AutoTokenizer, AutoModelForCausalLM, set_seed
import torch
from torch.profiler import profile, ProfilerActivity

vuiseng9

Install

https://github.com/state-spaces/mamba

Run

from transformers import MambaConfig, MambaForCausalLM, AutoTokenizer, AutoModelForCausalLM
import torch
from functools import partial
from collections import OrderedDict, defaultdict
import os

vuiseng9

openbmb/MiniCPM-V-2_6

https://huggingface.co/openbmb/MiniCPM-V-2_6

import torch
from PIL import Image
from transformers import AutoModelForCausalLM, AutoTokenizer
import requests

vuiseng9

Set 1	Set 2	Set 3	Least Character Word
Apex	Aurora	Atom	Apex
Blaze	Blitz	Bolt	Bolt
Cipher	Cascade	Cobalt	Cipher
Dynamo	Drift	Drift	Drift
Echo	Eclipse	Ember	Echo
Flux	Flare	Frost	Flux
Gravitas	Glide	Grit	Grit
Helix	Horizon	Haven	Helix