naive bayes,textfilter

textfilter.py

#coding=utf-8
import re
import os
import nltk
import math
from nltk.corpus import stopwords

stopword=stopwords.words('english')
traindir='./train'
testdir='./test'
    # func name: 采用文档频率的特征提取
    # input:traindir of file
    # ouput:dict of words

def Corpora(traindir):
    pattern=re.compile(r'.*(?=_)')
    txtdata = [f for f in os.listdir(traindir) if f.endswith(".txt")]#["a1.txt","a2.txt","b1.txt","b2.txt"]
    category=[pattern.match(i).group() for i in txtdata]
    corpora=zip(category,txtdata)
    return corpora
    #corpora=[('a', 'a1.txt'), ('a', 'a2.txt'), ('b', 'b1.txt'), ('b', 'b2.txt')]

def Class(corpora):
    dic={}
    for iter in corpora:
        # dic[iter[0]]=1 if (not iter[0] in dic.keys()) else dic[iter[0]]+=1
        if not iter[0] in dic:
            dic[iter[0]]=1
        else:
            dic[iter[0]]+=1
    return dic
    #dic={c1:frequent of class,c2:frequent of class}

def Normalize(document):#document="string"
    splitter=re.compile(r'\W*')
    porter=nltk.PorterStemmer()
    words=[porter.stem(w.lower()) for w in splitter.split(document) if w not in stopword]
    # words=[porter.stem(w.lower()) for w in splitter.split(document) ]
    return words

def Train(corpora,category):
    total=len(corpora)
    classfre = dict.fromkeys(category,1.0)
    feature={}#每个特征在各类文档中的概率 {特征:{类别:概率}}
    data=[]#保存预处理结果 [(c1,[t...]),(c2,[t...])]
    for classify,txt in corpora:
        do={}
        document=open(traindir+'/'+txt,'r').read() #需要close file?
        words=Normalize(document)
        data.append((classify,words))
        for w in words:
            if not w in feature:
                feature[w]=1
            else:
                if not w in do:
                    feature[w]+=1
            do[w]=1
    low = 1
    up = 7.0/10*total
    feature = {k :classfre.copy() for k,v in feature.iteritems() if v>low and v< up}
    for d in data:
        for w in d[1]:
            if w in feature:
                classfre[d[0]]+=1
                feature[w][d[0]]+=1
    for key in feature:
        for cate in feature[key]:
            feature[key][cate]=round(math.log( feature[key][cate] /classfre[cate]),5)
    return feature

def Predict(filename,category,featuredic):

    document=open(testdir+'/'+filename,'r').read()
    words=Normalize(document)
    maxscore=float('-inf')
    predict="somthing"
    totalcategorys=sum(category.values())+0.0
    for c in category.keys():
        score=round(math.log(category[c] / totalcategorys),5)
        for w in words:
            if w in featuredic:
                score+=featuredic[w][c]
        # print c,score
        if score>maxscore:
            predict=c
            maxscore=score
    return predict

def Test(testdir,category,featuredic):
    test=Corpora(testdir)
    test_cate={}
    for t in test:
        if not t[0] in test_cate:
            test_cate[t[0]]=[1.0,0.0]
        else:
            test_cate[t[0]][0]+=1
        cate=Predict(t[1],category,featuredic)
        if cate==t[0]:
            test_cate[t[0]][1]+=1
    sum=0.0
    right=0.0
    for t in test_cate:
        catetotal=test_cate[t][0]
        guessright=test_cate[t][1]
        sum+=catetotal
        right+=guessright
        print t, catetotal, guessright, round(guessright/catetotal*100,3)
    print "total",sum,right,round(right/sum*100,3)

    # total_test=len(test)+0.0
    # cate_right=0
    # for t in test:
    #     cate=Predict(t[1],category,featuredic)
    #     if cate==t[0]:
    #         cate_right+=1
    # print cate_right,total_test
    # print(round(cate_right / total_test *100,3))





if __name__ == '__main__':
    # traindir="."
    corp=Corpora(traindir)
    #corpora=[('a', 'a1.txt'), ('a', 'a2.txt'), ('b', 'b1.txt'), ('b', 'b2.txt')]
    category=Class(corp)
    #category={c1:frequency of class,c2:frequency of class}
    t=Train(corp,category.keys())
    Test(testdir,category,t)
    # p=Predict("Asia-Pacific_326.txt",category,t)



#
# V2
#
#coding=utf-8
import re
import os
import nltk
from nltk.corpus import stopwords

stopword=stopwords.words('english')
traindir='./train'
testdir='./test'
    # func name: 采用文档频率的特征提取
    # input:traindir of file
    # ouput:dict of words

def Corpora(traindir):
    pattern=re.compile(r'.*(?=_)')
    txtdata = [f for f in os.listdir(traindir) if f.endswith(".txt")]#["a1.txt","a2.txt","b1.txt","b2.txt"]
    category=[pattern.match(i).group() for i in txtdata]
    corpora=zip(category,txtdata)
    return corpora
    #corpora=[('a', 'a1.txt'), ('a', 'a2.txt'), ('b', 'b1.txt'), ('b', 'b2.txt')]

def Class(corpora):
    return set([iter[0] for iter in corpora])

def Normalize(document):#document="string"
    splitter=re.compile(r'\W*')
    porter=nltk.PorterStemmer()
    words=[porter.stem(w.lower()) for w in splitter.split(document) if w not in stopword]
    # words=[porter.stem(w.lower()) for w in splitter.split(document) ]
    return words

def Train(corpora):
    total=len(corpora)
    feature={}#每个特征在各类文档中的概率 {特征:{类别:概率}}
    data=[]#保存预处理结果 [(c1,[t...]),(c2,[t...])]
    for classify,txt in corpora:
        do={}
        document=open(traindir+'/'+txt,'r').read() #需要close file?
        words=Normalize(document)
        data.append((classify,words))
        for w in words:
            if not w in feature:
                feature[w]=1
            else:
                if not w in do:
                    feature[w]+=1
            do[w]=1
    low = 1
    up = total
    feature = {k :1 for k,v in feature.iteritems() if v>low and v< up}
    # print(data[0])
    trainset=[]
    for d in data:
        feature_set=feature.copy()
        for w in d[1]:
            if w in feature:
                feature_set[w]+=1
        trainset.append((feature_set,d[0]))
    return trainset

def GetFeature(document,feature):
    words=Normalize(document)
    for w in words:
        if w in feature:
            feature[w]+=1
    return feature

def Predict(filename,feature,classifier):
    document=open(testdir+'/'+filename,'r').read()
    feature_set=GetFeature(document,feature)
    return classifier.classify(feature_set)

def Test(testdir,feature,classifier):
    test=Corpora(testdir)
    total_test=len(test)+0.0
    cate_right=0
    for t in test:
        cate=Predict(t[1],feature,classifier)
        print cate,t[0]
        if cate==t[0]:
            cate_right+=1
    print cate_right,total_test
    print(round(cate_right / total_test *100,3))


if __name__ == '__main__':
    corp=Corpora(traindir)
    #corpora=[('a', 'a1.txt'), ('a', 'a2.txt'), ('b', 'b1.txt'), ('b', 'b2.txt')]
    train_set=Train(corp)
    #train_set=[({featuredic},'class'),...]
    classifier=nltk.NaiveBayesClassifier.train(train_set)
    Test(testdir,train_set[0][0],classifier)

v3:with pickle to store train and category