make lda training data

make_training_data.py

#!/usr/bin/python
# coding: utf8

import sys
import re
sub = re.compile('\n').sub

def hoge(text):
    fuga = sub('', text).split(',')
    foo = {}
    if len(fuga) < 5:
        return
    while(len(fuga) > 0):
        n = fuga.count(fuga[0])
        s = fuga[0]
        for i in range(n): fuga.remove(s)
        foo[s] = n
    print ' '.join([str(k) + ' ' + str(n) for k, n in foo.items()])

for line in sys.stdin:
    hoge(line)