mlp3.jl

mlp3.jl

sigmoid(x) = 1.0 ./ (1.0 + exp(-x))
dsigmoid(y) = y .* (1.0 - y)

type MLP3
    W¹::Matrix{Float64}
    b¹::Vector{Float64}
    W²::Matrix{Float64}
    b²::Vector{Float64}
end

function MLP3(nin, nhid, nout)
    MLP3(rand(nin, nhid), zeros(nhid), rand(nhid, nout), zeros(nout))
end

function fprop(net::MLP3, x)
    h = sigmoid(net.W¹' * x .+ net.b¹)
    y = sigmoid(net.W²' * h .+ net.b²)
    y, h
end

function bprop(net::MLP3, x, y, t, h; lr::Float64=0.1)
    Δᵒ = (y - t) .* dsigmoid(y)
    net.W², net.b² = net.W² - lr * (h * Δᵒ'), net.b² - lr * vec(sum(Δᵒ, 2))
    Δʰ = (net.W² * Δᵒ) .* dsigmoid(h)
    net.W¹, net.b¹ = net.W¹ - lr * (x * Δʰ'), net.b¹ - lr * vec(sum(Δʰ, 2))
end

function cost(net::MLP3, X, t)
    Y, _ = fprop(net, X)
    0.5 * sumabs2(Y - t) / size(X, 2)
end

function test_xor()
    srand(98765)

    X = reshape([0.,0,0,1,1,0,1,1], 2, 4)
    Y = reshape([0.,1,1,0], 1, 4)
    @assert size(X, 2) == size(Y, 2)

    nhid = 20
    epochs = 50000
    net = MLP3(2, nhid, 1)

    # Gradient decent (GD) or Stochastic GD
    sgd = true
    N = size(X, 2)
    for epoch in 1:epochs
        x, t = X, Y
        if sgd
            n = rand(1:N)
            x, t = X[:,n], Y[:,n]
        end

        y, h =  fprop(net, x)
        bprop(net, x, y, t, h)

        println("epoch #$(epoch), loss: $(cost(net, X, Y))")
    end

    Ŷ, _ = fprop(net, X)
    println(Ŷ)
    @assert round(Int, Ŷ) == Y
end

test_xor()