buffered hdf5 dataset in juli

buffered_hdf5.jl

using HDF5

"HDF5 appears to be inefficent for small writes, so this a simple buffer that
allows me to write to HDF5 only once per unit time (typically one second) to
limit the number of small writes."
mutable struct BufferedHDF5Dataset{T}
  ds::HDF5Dataset
  v::Vector{T}
  lasti::Int64 # last index in hdf5 dataset
  timeout_s::Float64 # interval in seconds at which to transfer data from v to ds
  endchannel::Channel{Bool} # stop writing if this channel is ready
  task::Task
end

function d_extend(d::HDF5Dataset, value::Vector, range::UnitRange)
  set_dims!(d, (maximum(range),))
	d[range] = value
	d
end

function write_to_hdf5(b::BufferedHDF5Dataset)
  r = b.lasti + (1:length(b.v))
  if length(r)>0
    d_extend(b.ds, b.v, r)
    empty!(b.v)
    b.lasti=last(r)
  end
  return
end

function Base.schedule(b::BufferedHDF5Dataset)
  b.task=@schedule begin
    while !isready(b.endchannel)
      write_to_hdf5(b)
      sleep(b.timeout_s)
    end
    write_to_hdf5(b)
  end
end
stop(b::BufferedHDF5Dataset) = put!(b.endchannel,true)
Base.wait(b::BufferedHDF5Dataset) = wait(b.task)

Base.write{T}(b::BufferedHDF5Dataset{T},x::T) = push!(b.v,x)
Base.write{T}(b::BufferedHDF5Dataset{T},x::Vector{T}) = append!(b.v,x)