NMT_vocab.py

from collections import Counter

files = ['train.to','train.from']

for name in files:
    words = []
    save_name = "vocab.{}".format(name.split('.')[1])
    print(save_name)
    with open(name, 'r') as f:
        data = f.read().split('\n')
        for l in data:
            for word in l.split(' '):
                if word != '':
                    words.append(word)
    count = Counter(words).most_common(50000)
    #print(count)

    with open(save_name, "a") as f:
        f.write("<unk>\n<s>\n</s>\n")
        for k,v in count:
            f.write(k+'\n')