eval_retrieval.py

import logging

from simpletransformers.retrieval import RetrievalModel, RetrievalArgs


logging.basicConfig(level=logging.INFO)
transformers_logger = logging.getLogger("transformers")
transformers_logger.setLevel(logging.WARNING)


eval_data = "data/nq-dev.json"

model_type = "custom"
model_name = "outputs"


# Create a TransformerModel
model = RetrievalModel(
    model_type=model_type,
    model_name=model_name,
)

results, *_ = model.eval_model(eval_data=eval_data)

print(results)