Scipy LBFGS

test_lbfgs.py

import os
import time
import numpy as np
import theano
import theano.tensor as T
from scipy.optimize import fmin_l_bfgs_b


def scipy():
    path = "."
    theano.config.floatX = "float64"
    x = np.loadtxt(os.path.join(path, "x.csv"), delimiter=",")
    x = theano.shared(x, name="x")
    y = np.loadtxt(os.path.join(path, "y.csv"), delimiter=",")
    y = theano.shared(y, name="y")
    n, d = x.shape.eval()
    w_all = np.loadtxt(os.path.join(path, "w.csv"), delimiter=",")
    w = T.dvector()
    W1 = T.reshape(w[:d * d], (d, d)).T
    W2 = T.reshape(w[-d:], (d, 1))
    h = T.tanh(T.dot(x, W1))
    pred = T.dot(h, W2).flatten()
    f = T.zeros_like(y)
    i0 = T.eq(y, 0).nonzero()
    i1 = T.eq(y, 1).nonzero()
    f = T.set_subtensor(f[i1], - T.nnet.softplus(-pred[i1]))
    f = T.set_subtensor(f[i0], - T.nnet.softplus(pred[i0]))
    f = - T.mean(f)
    g = T.grad(f, w)
    func = theano.function([w], [f, g])
    t = time.time()
    x, f, d = fmin_l_bfgs_b(func, w_all[:, 0], m=100, pgtol=1e-9, maxls=25, factr=10)
    print("Time:", time.time() - t)
    print("Final solution:")
    print(x)
    print("Final optimum:")
    print(f)
    print("True minimum")
    print(func(w_all[:, -1])[0])
    print("Stopping criteria:")
    print(d["task"])
    
if __name__ == '__main__':
    scipy()

w.csv

x.csv

y.csv

1
0
0
0
0
1
1
0
1
0