chainer_memory_usage.py

# GPU to use: Tesla K40
# Comments are GPU memory usage measured by nvidia-smi

import chainer, cupy, numpy

cupy.cuda.Device(1).use()  
a = cupy.array((), dtype=numpy.float32)  # 74MiB
# I have not investigated what components consume first 74MiB memory.

b = cupy.ones((1024, 1024), dtype=numpy.float32)  # 78MiB
# the total size of 1024x1024 float32 is 4MiB,
# so we can safely say almost all increment of memory usage is due to data attribute.
# If we create b by `cupy.uniform`, the memory usage increase by 12MiB (77MiB->89MiB).
# This is probably because CuPy sets up random generator inside it.

bb = chainer.Variable(b)  # 78MiB
# The overhead caused by wrapping ndarray by Variable seems negligible.

s = chainer.functions.sum(bb)  # 78MiB
s.backward()  # 82MiB
# The amount of increase is almost same as theoretical size of grad attribute (4MiB).
# So the overhead caused by adding grad attribute is also negligible.

c = [cupy.random.uniform(-1, 1, (1024, 1024), dtype=numpy.float32) for _ in xrange(1024)]  # 4189MiB
# memory usage per one ndarray = (4189 - 82) / 1024 = 4.01MiB