-
-
Save crhan/f04ddb7373533bae4d051271497e080e to your computer and use it in GitHub Desktop.
python serialization speed comparison
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
virtualenv --no-site-packages hodor | |
hodor/bin/pip install simplejson ujson cbor tnetstring msgpack-python | |
curl -s 'http://www.json-generator.com/api/json/get/cvfsLVmKiG?indent=2' > test.json | |
hodor/bin/python shootout.py |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
## python 3.6.6 on Mac Mojave | |
..::DUMPS::.. | |
serializer time time-diff size size-diff | |
json: 6.919172 100.00% 4978960 100.00% | |
simplejson: 103.331999 1493.42% 4978960 100.00% | |
ujson: tests failed: 'ujson' | |
msgpack: 2.662737 38.48% 3987280 80.08% | |
cbor: 113.861686 1645.60% 3997942 80.30% | |
marshal: 1.946263 28.13% 4364702 87.66% | |
pickle: 12.123323 175.21% 6912221 138.83% | |
pickle2: 12.438577 179.77% 6912221 138.83% | |
..::LOADS::.. | |
serializer time time-diff | |
json: 9.345431 100.00% | |
simplejson: 135.870550 1453.87% | |
ujson: tests failed: 'ujson' | |
msgpack: 3.977950 42.57% | |
cbor: 172.241193 1843.05% | |
marshal: 9.564885 102.35% | |
pickle: 5.318344 56.91% | |
pickle2: 5.242552 56.10% |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
""" | |
Forked from: https://gist.github.com/cactus/4073643 | |
""" | |
# pylint: disable=attribute-defined-outside-init | |
import functools | |
import json | |
import marshal | |
import pickle | |
import random | |
import sys | |
import cbor | |
import msgpack | |
import simplejson | |
import six | |
if six.PY2: | |
import cPickle # pylint: disable=import-error | |
try: | |
import ujson | |
except ImportError: | |
pass | |
class P(object): | |
pass | |
pickle2 = P() | |
pickle2.dumps = functools.partial(pickle.dumps, protocol=2) | |
pickle2.loads = pickle.loads | |
sys.modules['pickle2'] = pickle2 | |
if six.PY2: | |
cPickle2 = P() | |
cPickle2.dumps = functools.partial(cPickle.dumps, protocol=2) | |
cPickle2.loads = cPickle.loads | |
sys.modules['cPickle2'] = cPickle2 | |
# this is a big json data, length is | |
from serialize_performance import _fixture_data | |
testdict = _fixture_data() | |
def thrasher(serializer): | |
m = sys.modules[serializer] | |
dumper = getattr(m, 'dumps') | |
loader = getattr(m, 'loads') | |
def thrash(): | |
assert loader(dumper(testdict)) == testdict | |
return thrash | |
def thrash_loads(serializer): | |
m = sys.modules[serializer] | |
dumper = getattr(m, 'dumps') | |
loader = getattr(m, 'loads') | |
data = dumper(testdict) | |
def thrash(): | |
loader(data) | |
return thrash | |
def thrash_dumps(serializer): | |
m = sys.modules[serializer] | |
dumper = getattr(m, 'dumps') | |
def thrash(): | |
dumper(testdict) | |
return thrash | |
PERMUTATIONS = 100 | |
def print_round(name, tx, to, l=0, jl=0): | |
if l or jl: | |
print("%12s:\t%f\t%7.2f%%\t%d\t%7.2f%%" % | |
(name, tx, round((tx / to) * 100, 2), l, round( | |
(l / jl) * 100, 2))) | |
else: | |
print("%12s:\t%f\t%7.2f%%" % (name, tx, round((tx / to) * 100, 2))) | |
if __name__ == "__main__": | |
import timeit | |
tmodules = ["simplejson", "ujson", "msgpack", "cbor", "marshal", "pickle", | |
"pickle2"] | |
if six.PY2: | |
tmodules += ["cPickle", "cPickle2"] | |
if len(sys.argv) > 1: | |
tmodules = sys.argv[1].split(',') | |
print("..::DUMPS::..") | |
print("%12s\t%s\t\t%s\t%s\t%s" % ("serializer", "time", "time-diff", | |
"size", "size-diff")) | |
to = timeit.Timer( | |
"dumps()", | |
"from shootout import thrash_dumps;dumps=thrash_dumps('json')") | |
to = to.timeit(number=PERMUTATIONS) | |
jlen = float(len(json.dumps(testdict))) | |
print_round("json", to, to, jlen, jlen) | |
for x in tmodules: | |
try: | |
tx = timeit.Timer( | |
"dumps()", | |
"from shootout import thrash_dumps;dumps=thrash_dumps('%s')" % | |
x) | |
tx = tx.timeit(number=PERMUTATIONS) | |
m = sys.modules[x] | |
print_round(x, tx, to, len(m.dumps(testdict)), jlen) | |
except Exception as e: | |
print("%12s: tests failed: %s" % (x, e)) | |
print("") | |
print("..::LOADS::..") | |
print("%12s\t%s\t\t%s" % ("serializer", "time", "time-diff")) | |
to = timeit.Timer( | |
"loads()", | |
"from shootout import thrash_loads;loads=thrash_loads('json')") | |
to = to.timeit(number=PERMUTATIONS) | |
print_round("json", to, to) | |
for x in tmodules: | |
try: | |
tx = timeit.Timer( | |
"loads()", | |
"from shootout import thrash_loads;loads=thrash_loads('%s')" % | |
x) | |
tx = tx.timeit(number=PERMUTATIONS) | |
m = sys.modules[x] | |
print_round(x, tx, to) | |
except Exception as e: | |
print("%12s: tests failed: %s" % (x, e)) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment