simple statistics in a markdown corpora

statistics.py

#!/usr/bin/python
# -*- coding: utf-8 -*-

import os
import glob
import re
import codecs
import nltk
from random import *
from nltk.probability import *
from nltk.text import *

reader = nltk.corpus.reader.PlaintextCorpusReader("text/", '.*\.md', 
                                                  sent_tokenizer = nltk.data.LazyLoader('tokenizers/punkt/portuguese.pickle'),
                                                  encoding = "utf-8")

fd = None
for fileid in sample(reader.fileids(), 20):
    print "Processing", fileid
    coded = [ tok.encode('utf-8') for tok in reader.words(fileid) ]
    text = Text(coded)
    fdc = FreqDist(text)
    if fd == None:
        fd = fdc 
    else:
        fd = fdc + fd

for x in filter(lambda x: x[1] > 10, fd.items()):
    print "[", x[0], x[1], "]",