serialisation_benchmarking.py

import json
import pickle
import msgpack

TEST_REDIS = False

if TEST_REDIS:
    import redis
    rconn = redis.StrictRedis()
    dataset = [(('key:%d' % i), {'a': 1, 'b': list(range(100)), 'c': ('z' * 25)}) for i in range(1000000)]

def test_redis(serialize):
    if not TEST_REDIS:
        return
    rconn.flushall()
    for k, v in dataset:
        rconn.set(k, serialize(v))
    size = int(rconn.info()['used_memory'])
    print('%.2f MB' % (size / 1024 / 1024))

data1 = [1, 2, True, False, 'abcd']
data2 = dict((i, str(i) * 10) for i in range(20000))


print('pickle')
get_ipython().magic('timeit s = pickle.dumps(data1, protocol=-1); pickle.loads(s)')
get_ipython().magic('timeit s = pickle.dumps(data2, protocol=-1); pickle.loads(s)')
test_redis(pickle.dumps)
print('json')
get_ipython().magic('timeit s = json.dumps(data1); json.loads(s)')
get_ipython().magic('timeit s = json.dumps(data2); json.loads(s)')
test_redis(json.dumps)
print('msgpack')
get_ipython().magic('timeit s = msgpack.packb(data1); msgpack.unpackb(s)')
get_ipython().magic('timeit s = msgpack.packb(data2); msgpack.unpackb(s)')
test_redis(msgpack.dumps)

Environment

• Python 3.5.2 (ipython 5.1.0)
• Ubuntu 16.04 on Macbook Air 7,2

Results

pickle

The slowest run took 6.75 times longer than the fastest. This could mean that an intermediate result is being cached.
100000 loops, best of 3: 2.27 µs per loop
100 loops, best of 3: 6.86 ms per loop
389.94 MB

json

The slowest run took 8.20 times longer than the fastest. This could mean that an intermediate result is being cached.
100000 loops, best of 3: 6.85 µs per loop
10 loops, best of 3: 20.5 ms per loop
512.02 MB

msgpack

The slowest run took 24.26 times longer than the fastest. This could mean that an intermediate result is being cached.
100000 loops, best of 3: 2.23 µs per loop
100 loops, best of 3: 6.83 ms per loop
237.36 MB