README.md

You should rather take a look at this blog post.

bfgs.py

def bfgs(x0, fun, grad, n_iter, eps=1e-5, n_linesearch=10, return_path=False):
    # https://en.wikipedia.org/wiki/Broyden%E2%80%93Fletcher%E2%80%93Goldfarb%E2%80%93Shanno_algorithm
    x = np.copy(x0)
    path = [np.copy(x0)]

    n_params = x.shape[0]
    B = np.eye(n_params)

    old_old_f = None
    old_f = None

    g = grad(x)
    for _ in range(n_iter):
        p = np.linalg.solve(B, -g)

        """
        from scipy.optimize import line_search
        alpha, n_fevals, n_gevals, old_f, old_old_f, g_new = line_search(fun, grad, x, p, g, old_f, old_old_f)
        """
        # Simple line search
        alphas = np.logspace(-3, 4, n_linesearch)
        f = np.array([fun(x + alpha * p) for alpha in alphas])
        alpha = alphas[np.argmin(f)]

        s = alpha * p
        x += s

        prev_g = g
        g = grad(x)
        if prev_g is not None:
            y = g - prev_g
            B += np.outer(y, y) / np.dot(y, s) - B.dot(s).dot(s.dot(B)) / s.dot(B).dot(s)

        path.append(np.copy(x))

        if np.linalg.norm(g) <= eps:
            break

    if return_path:
        return x, path
    else:
        return x