Convert R data frame to msgpack stream

as_msgpack_stream.R

devtools::install_github("crowding/msgpack-r")

library(msgpack)

x <- data.frame(a=1:3, b=4:6)

conn <- gzfile("test.msgpack.gz", open="w+b")
apply(x, 1, function(x) {writeMsg(c(x, time=as.integer(Sys.time())), conn)})
flush(conn)
close(conn)

read_msgpack_stream.py

import gzip

import msgpack

with gzip.GzipFile("test.msgpack.gz", "rb") as f:
    unpacker = msgpack.Unpacker(f, raw=False)
    for u in unpacker:
        print(u)

# {'a': 1, 'b': 4, 'time': 1572094906}
# {'a': 2, 'b': 5, 'time': 1572094906}
# {'a': 3, 'b': 6, 'time': 1572094906}