convert_to_spacy.py

import sys
import json


fo = open(sys.argv[1], "r")

lines = fo.readlines()


for line in lines:
    line =json.loads(line)
    if "labels" in line:
    	line["entities"] = line.pop("labels")
    else:
	    line["entities"] = []

    tmp_ents = []
    for e in line["entities"]:
        if e[2] in ['RISK', 'ORG', 'GPE', 'DATE', 'LAW', 'CARDINAL', 'MONEY', 'PRODUCT', 'ORDINAL', 'PERCENT', 'LOC', 'NORP', 'EVENT', 'WORK_OF_ART', 'FAC', 'PERSON', 'TIME']:
     	    tmp_ents.append({"start": e[0], "end": e[1], "label": e[2]})
        
        line["entities"] = tmp_ents

    if (len(line["text"]) > 5):
        print (json.dumps({"entities": line["entities"], "text": line["text"]}))