cgt_memory.py

import cgt
import cgt.nn as nn
import numpy as np
from scipy.stats import norm

def gaussian_density(x, mu, sigma):
    return cgt.exp(-cgt.square(x - mu) / 2 / cgt.square(sigma)) \
        / cgt.sqrt(2 * np.pi) / sigma

var_mu = nn.parameter(np.array(0.5))
var_log_sigma = nn.parameter(np.array(0.5))

x = cgt.scalar('x')

mu = np.array(0.3)
sigma = np.array(1.)

q_density = gaussian_density(x, var_mu, cgt.exp(var_log_sigma))

KL = cgt.log(q_density) \
    - cgt.log(gaussian_density(x, mu, sigma))

params = nn.get_parameters(KL)

#note that baseline acts only in computing the gradients,
#not the KL itself
KL_grad = [cgt.grad(q_density, [p])[0] * KL for p in params]

KL = cgt.function(inputs=[x], outputs=[KL])

N = 10000000

import gc

def eval(N):
    x = norm(loc=var_mu.op.get_value(),
        scale=np.exp(var_log_sigma.op.get_value())).rvs(size=N)
    l = 0.
    for i in xrange(N):
        l += KL(x[i])[0] / N
        if i % 1000 == 0:
            gc.collect()
    return l

l = eval(N)
print l