Script to join text from JSON files for training GPT-2

json2txt.py

import json
import os
import argparse

parser = argparse.ArgumentParser()
parser.add_argument('input_dir', help='Directory with wiki json files')
parser.add_argument('output', help='Txt file output')
args = parser.parse_args()

for filename in os.listdir(args.input_dir):
    if not filename.startswith('wiki'):
        continue

    path = os.path.join(args.input_dir, filename)
    with open(path, 'r') as fin, open(args.output, 'a') as fout:
        for line in fin:
            data = json.loads(line)

            # remove non-processed templates
            text = data['text'].replace('()', '')

            fout.write(text)
            fout.write('<|endoftext|>\n')