convert livedoor news corpus to spacy gold jsonl

ldcc2spacygold.py

from pathlib import Path
import itertools as it
import copy
import srsly
from tqdm.notebook import tqdm


labels = [
    "movie-enter",
    "it-life-hack",
    "kaden-channel",
    "topic-news",
    "livedoor-homme",
    "peachy",
    "sports-watch",
    "dokujo-tsushin",
    ".ipynb_checkpoints",
    "smax",
]

base_cats = dict(zip(labels, it.repeat(0)))


def file2text(fpath):
    with fpath.open() as f:
        texts = f.read().split("\n")[2:]
        texts = filter(lambda x: x != "", texts)
    return "\n".join(texts)


def gen_golds(labels):
    for d in tqdm(labels):
        cats = copy.copy(base_cats)
        cats[d] = 1
        for f in tqdm(Path(d).glob("*.txt")):
            text = file2text(f)
            yield text, {"cats": cats}


srsly.write_jsonl("gold.jsonl", gen_golds(labels))