auto eval openclip

auto_eval_openclip.md

Change paths then run keep_evaling.sh

Will keep evaluating openclip during training

to send to wandb, can also run while [ 1 ]; do python3 eval_to_wandb.py; sleep 300; done

eval.sh

ev=eval_$1
if [ -f "$ev" ]; then
  true
else
  echo "$ev does not exist."
  python -m training.main \
    --imagenet-val /fsx/rom1504/imagenetval/imagenet_validation \
     --model ViT-H-14 \
    --pretrained /fsx/rom1504/open_clip/src/logs/2022_07_30-09_18_18-model_ViT-H-14-lr_0.0005-b_96-j_12-p_amp/checkpoints/$1 &> eval_$1
fi

eval_to_wandb.py

import glob
import wandb

files = list(glob.glob("eval_epoch_*.pt"))
metrics = []
for filename in files:
    epoch = int(filename.split("_")[-1].split(".")[0])
    f = open(filename, "r")
    c = f.read()
    good =[l for l in c.split("\n") if "imagenet-zero" in l]
    if len(good) != 1:
        continue
    top5 = float(good[0].split("\t")[1].split(" ")[-1])
    top1 = float(good[0].split("\t")[0].split(" ")[-1])
    metrics.append([epoch, top1, top5])

metrics.sort(key=lambda x:x[0])
print(metrics)

wandb.init(project="eval_openclip", id="pe5tibnh", resume="allow")

for epoch, top1, top5 in metrics:
    wandb.log({'top1': top1, 'top5': top5}, step=epoch)

keep_evaling.sh

while [ 1 ]
do
        for i in `ls /fsx/rom1504/open_clip/src/logs/2022_07_30-09_18_18-model_ViT-H-14-lr_0.0005-b_96-j_12-p_amp/checkpoints/`
        do
                bash eval.sh $i
        done
sleep 300
done