evaluate model

evaluate.py

def evaluate(tokenizer, textcat, texts, cats):
    """
    Evaluate the performance of TextCategoriser prediction
    Calculate accuracy, f1 score, precision, recall
    parameters:
        nlp: object - spacy
        textcat: TextCategoriser
        texts : input text to be evaluated
        cats : input label
    """
    docs = (tokenizer(text) for text in texts)
    tp = 0.0  # True positives
    fp = 1e-8  # False positives
    fn = 1e-8  # False negatives
    tn = 0.0  # True negatives

    for i, doc in enumerate(textcat.pipe(docs)):
        gold = cats[i]
        for label, score in doc.cats.items():
            if label not in gold:
                continue
            if label == "NEGATIVE":
                continue
            if score >= 0.5 and gold[label] >= 0.5:
                tp += 1.0
            elif score >= 0.5 > gold[label]:
                fp += 1.0
            elif score < 0.5 and gold[label] < 0.5:
                tn += 1
            elif score < 0.5 <= gold[label]:
                fn += 1
    # calculate metrics
    accuracy = (tp + tn) / (tp + fp + fn + tn)
    precision = tp / (tp + fp)
    recall = tp / (tp + fn)
    if (precision + recall) == 0:
        f_score = 0.0
    else:
        f_score = 2 * (precision * recall) / (precision + recall)

    return {"textcat_a": accuracy, "textcat_p": precision, "textcat_r": recall, "textcat_f": f_score}