Evaluation of different BERT models on SQuADv1.1

bert-squad-eval.md

Evaluation of different BERT models on SQuADv1.1

https://github.com/huggingface/transformers

# following has been validated with transformers v4.18

# 24 Layers
# https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad
model=bert-large-uncased-whole-word-masking-finetuned-squad

# 12 Layers
# https://huggingface.co/vuiseng9/bert-base-uncased-squad
model=vuiseng9/bert-base-uncased-squad

# 6 Layers
# https://huggingface.co/distilbert-base-cased-distilled-squad
model=distilbert-base-cased-distilled-squad

batch_size=64
cd transformers/examples/pytorch/question-answering/
python run_qa.py \
    --model_name_or_path $model \
    --dataset_name squad \
    --do_eval \
    --per_device_eval_batch_size $batch_size \
    --max_seq_length 384 \
    --doc_stride 128 \
    --output_dir /tmp/eval-$model \
    --overwrite_output_dir

plot of events per QPS

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

qps_list = [50, 60, 70, 80, 300, 1000]
latency_bound = 15

ncol=2
nrow=len(qps_list)//ncol + len(qps_list)%ncol
fig, ax = plt.subplots(nrows=nrow, ncols=ncol, figsize=(10,10))

nevent=10

for axid, qps in enumerate(qps_list):
    r=axid//ncol
    c=axid%ncol
    
    mean_interarrival_duration = 1/qps * 1000
    
    event_interarrival = np.random.exponential(mean_interarrival_duration, nevent)
    
    event_begin_ts = np.cumsum(event_interarrival)
    
    event_constraint_ts = event_begin_ts + latency_bound
       
    ax[r, c].barh(range(nevent),  event_constraint_ts-event_begin_ts, left=event_begin_ts)
    ax[r, c].set_title("{} qps\n{:.0f}ms (mean_interarrival)".format(qps, mean_interarrival_duration))
    ax[r, c].set_yticks(range(nevent))
    ax[r, c].set_yticklabels(['e{}'.format(i) for i in range(nevent)])

fig.tight_layout()
fig.suptitle("MLPerf Image Classification Latency Bound {}ms ({:.0f}qps)".format(latency_bound, 1/latency_bound*1000), y=1.05, fontsize=15)