Just messing around with memmap arrays in response to a mailing list question.

memmap_sum.py

N = 2*10**7
chunk_size = 100000

farr1 = 'scratch/arr1'
farr2 = 'scratch/arr2'

arr1 = np.memmap(farr1, dtype='uint8', mode='w+', shape=(N, 4))
arr2 = np.memmap(farr2, dtype='uint8', mode='w+', shape=(N, 4))

for i in xrange(0, N, chunk_size):
    arr1[i:i+chunk_size] = np.random.randint(2, size=(chunk_size, 4)).astype(np.uint8)
    arr2[i:i+chunk_size] = np.random.randint(2, size=(chunk_size, 4)).astype(np.uint8)

del arr1
del arr2

arr1 = np.memmap(farr1, mode='r', dtype='uint8', shape=(N,4))
arr2 = np.memmap(farr2, mode='r', dtype='uint8', shape=(N,4))


equal = np.logical_and(arr1[:chunk_size], 
                       arr2[:chunk_size]).sum(0)

for i in xrange(chunk_size, N, chunk_size):
    equal += np.logical_and(arr1[i:i+chunk_size], 
                            arr2[i:i+chunk_size]).sum(0)