StrPack test

make_data.py

import numpy as np
import mmap
import os

N = 5
dtype = [('a', np.float64), ('b', np.int32), ('c', np.float32), ('d', np.int16), ('e', 'S4')]
names = [a[0] for a in dtype]

columns = [np.random.normal(size=(N,)).astype(np.float64), 
           np.random.normal(size=(N,)).astype(np.int32), 
           np.random.normal(size=(N,)).astype(np.float32), 
           np.random.normal(size=(N,)).astype(np.int16),
           np.empty(N, dtype='S4')]

columns[-1][:] = 'abcd'

print 'Data:'
for i, col in enumerate(columns):
    print '{}: {}'.format(names[i], col)

print 'Itemsizes:'
for i, col in enumerate(columns):
    print '{}: {}'.format(names[i], col.itemsize)

print ''
print 'Record size: {}'.format(sum([col.itemsize for col in columns]))

size = columns[0].size
nbytes = sum([col.size * col.itemsize for col in columns])

# Create file
os.system('rm test1.dat')
os.system('dd bs={nbytes} seek={nbytes} count=0 if=/dev/null of=test1.dat'.format(nbytes=nbytes))


_npdata = np.rec.array(np.zeros(1, dtype))
dtype = _npdata.dtype

# create an empty file to be memory-mapped
f = open('test1.dat', 'a+')

print 'Total bytes: ', nbytes

# memory map the file
charmap = mmap.mmap(f.fileno(), nbytes)

# lay out a numpy array in that memory
npmap = np.ndarray(size, dtype=dtype, buffer=charmap)

# copy the data to memory mapped data, thus writing the file
for i, name in enumerate(names):
    npmap[name][:] = columns[i][:]
    f.flush()

# closing the file causes it to flush to disk, and releases the memory map
f.close()

packed_struct.jl

using StrPack

bitstype 32 Char4

@struct immutable TestType
    a::Float64
    b::Int32
    c::Float32
    d::Int16
    e::ASCIIString(4)
    #=e::Array{Uint8,1}(4)=#
end

data = Array(TestType, (5,))

f = open("test1.dat", "r")
for k = 1:5
    data[k] = unpack(f, TestType, {"a"=>8, "b"=>4,"c"=>4, "d"=>2, "e"=>4}, align_packed, :NativeEndian)
    println(data[k])
end