mika_nn.py

# coding: utf-8

import numpy as np
import sys
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.cross_validation import train_test_split

def logistic_function(mat):
    return 1/(1+np.exp(-mat))


# ユニットの活性化関数の微分を引数(行列)に適用
def df(u_l):
    return logistic_function(u_l)*(1-logistic_function(u_l))


def forward(network, X):
    #  ユニットの入力と出力を初期化
    U = [X.toarray()]
    Z = [X.toarray()]
    for i, (w, b) in enumerate(network):
        u_l = w.dot(Z[i]) + b.dot(np.ones([1, X.shape[1]]))
        U.append(u_l)
        Z.append(logistic_function(u_l))
    predict_Y = Z[-1]
    return U, Z, predict_Y


def backpropagation(network, delta_L, U):
    delta = [delta_L]
    for l in range(len(network)-1, -1,  -1):
        w = network[l][0]
        u = U[l]
        delta_l = df(u) * w.transpose().dot(delta[0])
        delta.insert(0, delta_l)
    return delta


# 並列化できるがここでは逐次的に
def update_network(network, N, delta, Z):
    epsion = 0.1 # 学習率

    for l in range(len(network)-1, -1,  -1):
        w, b = network[l]
        w -= (1/N)*(epsion * delta[l+1].dot(Z[l].transpose()))
        b -= (1/N)*(epsion * delta[l+1].dot(np.ones([N, 1])))
        network[l] = [w, b]
    return network


# 対数尤度関数
def err(D, Y):
    err = 0.0
    for i in range(Y.shape[1]):
        err += D[i] * np.log2(Y[0, i]) + ((1-D[i]) * np.log2(1-Y[0, i]))

    return -err/len(D)

# testデータに対する正解率
def err_rate(D, Y):
   c = 0
   for i in range(Y.shape[1]):
       r = 0
       if Y[0, i] >= 0.5:
           r = 1
       if D[i] == r:
           c += 1
   return c/Y.shape[1]


fname = sys.argv[1]

D = [] # label
doc = [] # raw data

with open(fname) as f:
    cv = CountVectorizer()
    for l in f:
        data = l.strip().split(" ", 1)
        D.append(int(data[0]))
        doc.append(data[1])
    D = np.array(D)

cv = CountVectorizer(ngram_range=(1, 2),token_pattern='(?u)\\b\\w+\\b')
cv.fit(doc)

train_raw_docs, test_raw_docs, train_D, test_D = train_test_split(doc , D, test_size=0.15, random_state=42) # test data rate 0.15

train_Docs = cv.transform(train_raw_docs).transpose()
test_Docs = cv.transform(test_raw_docs).transpose()

del D, doc

print(train_Docs.shape)
hidden_layer = [300, 150, 1]  # 各層のユニットの数

# ネットワークの重みとバイアスを初期化
net = []
for h in range(len(hidden_layer)):
    if h:
        W = np.random.rand(hidden_layer[h], hidden_layer[h-1]) - 0.5
    else:
        W = np.random.rand(hidden_layer[h], train_Docs.shape[0])-0.5
    b = np.random.rand(hidden_layer[h], 1) - 0.5
    net.append([W, b])


test_result = []
err_result = []
for i in range(1112):

    # create minibatch
    randints = np.random.randint(0, train_Docs.shape[1], size=30)
    X = train_Docs[:, randints]
    mini_D = train_D[randints]

    U, Z, predict_Y = forward(net, X)

    delta_L = np.array(-mini_D + predict_Y)
    delta = backpropagation(net, delta_L, U)
    net = update_network(net, X.shape[1], delta, Z)
    
    
    print(err(mini_D, predict_Y))
    print(err_rate(test_D, forward(net, test_Docs)[2]))