evalmodel.py

from __future__ import unicode_literals, print_function
import re
import spacy
import pandas as pd
from spacy import displacy
import os
from collections import defaultdict
import itertools
import plac
import random
from pathlib import Path
from spacy.util import minibatch, compounding
import math
from spacy.pipeline import Sentencizer
from itertools import compress

def test_eval_model(model, TEST_DATA, viz = True, test_text="It is is designed to interoperate with the Python numerical and scientific libraries NumPy and SciPy."):
    """Make predictions and compute evaluation
        Args:
            model (str): path or a name of the model
            TEST_DATA (obj): object with sentences and targets positions, the same as for spacy train_spacy_NER()
            viz (bool): True/False if True use displacy to highlight the target in the test sentence 
            test_text (str): a test sentence to be displayed and targets shown
        Returns:
            dico (dictionnary) of model name, precision, recall and f1 score
            df_metrics (pd.DataFrame) : pandas data frame with evaluation metrics, usefull for manual evaluation
    """
    nlp = spacy_load_light(model)
    res= [predict_ner_model_internal(x, nlp) for x in TEST_DATA]
    precision, recall, f1_score, df_metrics = evaluate_model_int(pd.DataFrame(res), "predicted_targets","true_targets")
    dico = {"name": model , "precision" : precision, "recall" : recall,"f1_score": f1_score}
    doc= nlp(test_text)
    if viz:
        for ent in doc.ents:
            print(ent.label_, ent.text)
            colors={"TARGET":"Yellow"}
            options = {"ents": ["TARGET"], "colors": colors}
            displacy.render(doc, style='ent', jupyter=True, options=options)
    return(dico, df_metrics)

def predict_ner_model_internal(example, nlp):
    """Takes an item from the data list (same format as spacy train) and predicts the target
    Args:
        example (obj): an item in spacy format
        nlp (obj) : model loaded with spacy.load
    Returns:
        dictionnary with sentence, true targets and predicted targets
    """
    text= example[0]
    targets = example[1]["entities"]
    doc= nlp(text)
    tt=[]
    for toupt in targets:
        start = toupt[0]
        end = toupt[1]
        target = text[start:end]
        tt.append(target.lower())
    pt = []
    for ent in doc.ents:
        pt.append(ent.text.lower())
    tt=sorted(list(set(tt)))
    pt=sorted(list(set(pt)))
    return(dict({"sentence": text, "predicted_targets": pt, "true_targets": tt}))

def evaluate_model_int(df, pt, tt):
    """Prepare data frame for the evaluation

    """
    df["eval"] = df.apply(lambda x : evaluate_pred(x, pt,tt), axis=1)
    df["pos"] = df.apply(lambda x : is_positive(x, tt), axis=1)
    df["TP"]=None
    df["TN"]=None
    df["FN"]=None
    df["FP"]=None
    df["eval"] = df.apply(lambda x : evaluate_pred(x, pt,tt), axis=1)
    df["pos"] = df.apply(lambda x : is_positive(x, tt), axis=1)
    return(compute_metrics(df))


def evaluate_pred(row, pt,tt):
    """Verify if predicted target is equal to the true traget
        Args:
            row (pd.DataFrame) : row of pandas table
            pt (str) : predicted target column name
            tt (str) : true target column name
        Returns: 
            0 if pt==tt 
            1 if pt !=tt
    """
    if row[pt] == row[tt]:
        return(1)
    else:
        return(0)
    
def is_positive (row, tt):
    """Verify if the sentence has a target (true target)
        Args:
            row (pd.DataFrame) : row of pandas table
            tt (str) : true target column name
        Returns: 
            0 if there is no target
            1 if there is a target
    """
    if len(row[tt]) > 0 :
        return(1)
    else:
        return(0)


def compute_par_metrics(row):
    """Compute TP, TN, FP,FN  for each row
        Args:
            row (pd.DataFrame) : row of pandas table
        Returns:
            row (pd.DataFrame) with extra columns "TP", "TN", "FP", "FN"
    """
    if row["eval"]==1 and row["pos"]==1:
        row["TP"] = 1   
    elif row["eval"]==1 and row["pos"]==0:
        row["TN"] = 1
    elif row["eval"]==0 and row["pos"]==0:
        row["FP"] = 1
    elif row["eval"]==0 and row["pos"]==1:
        row["FN"] = 1
    return(row)


def compute_metrics(df):
    """Compute final metrics for evaluated sample 
        Args:
            df (pd.DataFrame) : data frame with true and predeicted targets 
        Retruns:
            precision (float)
            recall (float) 
            f1_score (float)
            df_metrics (pd.DataFrame) with extra columns "TP", "TN", "FP", "FN"
    """
    df_metrics = df.apply(lambda x: compute_par_metrics(x), axis=1)
    TP_s = df_metrics["TP"].fillna(0).sum()
    TN_s = df_metrics["TN"].fillna(0).sum()
    FN_s = df_metrics["FN"].fillna(0).sum()
    FP_s = df_metrics["FP"].fillna(0).sum()

    precision  = TP_s / (TP_s +FP_s)
    recall = TP_s / (TP_s +FN_s)
    f1_score = 2 * (precision*recall/(precision+recall))

    print("precision : ", precision)
    print("recall : ", recall)
    print("f1_score : ", f1_score)

    return(precision, recall, f1_score, df_metrics)