Blocks Batch Normalization Testing

test_batch_norm.py

# You are supposed to run this after the `Batch Normalization Tutorial`
import theano
import numpy as np

outputs = VariableFilter(
             bricks=mlp.linear_transformations, roles=[OUTPUT])(cg_bn.variables)

f = []
for o, g, b in zip(outputs, gammas, betas):
    f.append(
        theano.function([x], o*g + b)
    )
    
data_train = DataStream(
                mnist_train,
                iteration_scheme=SequentialScheme(mnist_train.num_examples, 100))

means = [0 for w in f]
sqrs = [0 for w in f]
for b in data_train.get_epoch_iterator():
    for i, w in enumerate(f):
        out = w(b[0])
        means[i] += out.sum(axis=0)
        sqrs[i] += (out**2).sum(axis=0)

N = mnist_train.num_examples
for i, (m, s) in enumerate(zip(means, sqrs)):
    means[i] = m/N
    sqrs[i] = s/N - means[i]**2
    means[i] = theano.shared(means[i].astype('float32'))
    sqrs[i] = theano.shared(sqrs[i].astype('float32'))
    
epsilon = 1e-5
gammas_pop = [gamma[0] / tensor.sqrt(v + epsilon) for gamma, v in zip(gammas, sqrs)]
betas_pop = [beta[0] - (gamma[0] * m / tensor.sqrt(v) + epsilon) 
             for beta, gamma, m, v in zip(betas, gammas, means, sqrs)]
             
use_population = dict((v, True) for v in variables)
cg_bn_test = apply_batch_normalization(
                cg, variables, gammas_pop, betas_pop, use_population=use_population, epsilon=1e-5)

final_func = theano.function([x, y], cg_bn_test.variables[-1])

# Finally
data_test = DataStream(
                 mnist_test,
                 iteration_scheme=SequentialScheme(
                     mnist_test.num_examples, 500))

m = 0
for i, (feats, targ) in enumerate(data_test.get_epoch_iterator()):
    m += final_func(feats, targ)
m /= i+1
print m