Last active
February 28, 2020 05:43
-
-
Save dropwhile/4073643 to your computer and use it in GitHub Desktop.
python serialization speed comparison
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| virtualenv --no-site-packages hodor | |
| hodor/bin/pip install simplejson ujson cbor tnetstring msgpack-python | |
| curl -s 'http://www.json-generator.com/api/json/get/cvfsLVmKiG?indent=2' > test.json | |
| hodor/bin/python shootout.py |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ## python 2.7.7 on centos 6.6 | |
| ..::DUMPS::.. | |
| json: 3.068149 100.00% 90460 100.00% | |
| simplejson: 5.616200 183.05% 90460 100.00% | |
| ujson: 1.875687 61.13% 89261 98.67% | |
| tnetstring: 1.162987 37.91% 90207 99.72% | |
| msgpack: 1.110323 36.19% 86798 95.95% | |
| cbor: 0.678674 22.12% 86778 95.93% | |
| marshal: 1.896508 61.81% 91600 101.26% | |
| pickle: 42.890038 1397.91% 91162 100.78% | |
| pickle2: 47.861820 1559.96% 87499 96.73% | |
| cPickle: 7.550282 246.09% 91452 101.10% | |
| cPickle2: 1.689119 55.05% 85602 94.63% | |
| ..::LOADS::.. | |
| serializer time time-diff | |
| json: 7.419880 100.00% | |
| simplejson: 2.558231 34.48% | |
| ujson: 2.890960 38.96% | |
| tnetstring: 1.000095 13.48% | |
| msgpack: 0.858918 11.58% | |
| cbor: 0.947492 12.77% | |
| marshal: 0.975319 13.14% | |
| pickle: 35.305399 475.82% | |
| pickle2: 20.916351 281.90% | |
| cPickle: 4.277538 57.65% | |
| cPickle2: 1.347456 18.16% | |
| ## [PyPy 2.4.0 with GCC 4.8.2] on centos 6.6 | |
| ## note: tests probably don't run long enough to get much JIT benefit | |
| ..::DUMPS::.. | |
| serializer time time-diff size size-diff | |
| json: 4.242018 100.00% 90460 100.00% | |
| simplejson: 17.838388 420.52% 90460 100.00% | |
| ujson: tests failed: 'ujson' | |
| tnetstring: 6.875353 162.08% 90207 99.72% | |
| msgpack: 3.891492 91.74% 86798 95.95% | |
| cbor: 4.522691 106.62% 86778 95.93% | |
| marshal: 5.291648 124.74% 91600 101.26% | |
| pickle: 22.375729 527.48% 95288 105.34% | |
| pickle2: 9.940469 234.33% 89020 98.41% | |
| cPickle: 25.145335 592.77% 95291 105.34% | |
| cPickle2: 10.484767 247.16% 89023 98.41% | |
| ..::LOADS::.. | |
| serializer time time-diff | |
| json: 3.343928 100.00% | |
| simplejson: 15.526334 464.31% | |
| ujson: tests failed: 'ujson' | |
| tnetstring: 67.718456 2025.12% | |
| msgpack: 1.826920 54.63% | |
| cbor: 6.167334 184.43% | |
| marshal: 4.911089 146.87% | |
| pickle: 9.324833 278.86% | |
| pickle2: 7.627817 228.11% | |
| cPickle: 8.596466 257.08% | |
| cPickle2: 7.294959 218.16% | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import sys | |
| import random | |
| import json | |
| import simplejson | |
| try: | |
| import ujson | |
| except ImportError: | |
| pass | |
| import tnetstring | |
| import marshal | |
| import pickle | |
| import cPickle | |
| import cbor | |
| import msgpack | |
| import functools | |
| from tnetstring.tests.test_format import FORMAT_EXAMPLES, get_random_object | |
| class P(object): | |
| pass | |
| pickle2 = P() | |
| pickle2.dumps = functools.partial(pickle.dumps, protocol=2) | |
| pickle2.loads = pickle.loads | |
| sys.modules['pickle2'] = pickle2 | |
| cPickle2 = P() | |
| cPickle2.dumps = functools.partial(cPickle.dumps, protocol=2) | |
| cPickle2.loads = cPickle.loads | |
| sys.modules['cPickle2'] = cPickle2 | |
| try: | |
| import __pypy__ | |
| tnetstrng = tnetstring | |
| tnetstring = P() | |
| tnetstring.dumps = functools.partial(tnetstrng.dumps, encoding="utf8") | |
| tnetstring.loads = functools.partial(tnetstrng.loads, encoding="utf8") | |
| sys.modules['tnetstring'] = tnetstring | |
| except: | |
| pass | |
| # test.json generated from http://www.json-generator.com/ | |
| # http://www.json-generator.com/api/json/get/cvfsLVmKiG?indent=2 | |
| testdict = simplejson.load(open('test.json','rb')) | |
| def thrasher(serializer): | |
| m = sys.modules[serializer] | |
| dumper = getattr(m, 'dumps') | |
| loader = getattr(m, 'loads') | |
| def thrash(): | |
| assert loader(dumper(testdict)) == testdict | |
| return thrash | |
| def thrash_loads(serializer): | |
| m = sys.modules[serializer] | |
| dumper = getattr(m, 'dumps') | |
| loader = getattr(m, 'loads') | |
| data = dumper(testdict) | |
| def thrash(): | |
| loader(data) | |
| return thrash | |
| def thrash_dumps(serializer): | |
| m = sys.modules[serializer] | |
| dumper = getattr(m, 'dumps') | |
| def thrash(): | |
| dumper(testdict) | |
| return thrash | |
| PERMUTATIONS=10000 | |
| def print_round(name, tx, to, l=0, jl=0): | |
| if l or jl: | |
| print "%12s:\t%f\t%7.2f%%\t%d\t%7.2f%%" % ( | |
| name, tx, | |
| round((tx / to) * 100,2), | |
| l, | |
| round((l / jl) * 100,2)) | |
| else: | |
| print "%12s:\t%f\t%7.2f%%" % ( | |
| name, tx, | |
| round((tx / to) * 100,2)) | |
| if __name__ == "__main__": | |
| import timeit | |
| tmodules = ("simplejson", "ujson", "tnetstring", "msgpack", "cbor", | |
| "marshal", "pickle", "pickle2", "cPickle", "cPickle2") | |
| if len(sys.argv) > 1: | |
| tmodules = sys.argv[1].split(',') | |
| print "..::DUMPS::.." | |
| print "%12s\t%s\t\t%s\t%s\t%s" % ("serializer","time","time-diff","size", "size-diff") | |
| to = timeit.Timer( | |
| "dumps()", | |
| "from shootout import thrash_dumps;dumps=thrash_dumps('json')") | |
| to =to.timeit(number=PERMUTATIONS) | |
| jlen = float(len(json.dumps(testdict))) | |
| print_round("json", to, to, jlen, jlen) | |
| for x in tmodules: | |
| try: | |
| tx = timeit.Timer( | |
| "dumps()", | |
| "from shootout import thrash_dumps;dumps=thrash_dumps('%s')" % x) | |
| tx = tx.timeit(number=PERMUTATIONS) | |
| m = sys.modules[x] | |
| print_round(x, tx, to, len(m.dumps(testdict)), jlen) | |
| except Exception as e: | |
| print "%12s: tests failed: %s" % (x, e) | |
| print "" | |
| print "..::LOADS::.." | |
| print "%12s\t%s\t\t%s" % ("serializer","time","time-diff") | |
| to = timeit.Timer( | |
| "loads()", | |
| "from shootout import thrash_loads;loads=thrash_loads('json')") | |
| to =to.timeit(number=PERMUTATIONS) | |
| print_round("json", to, to) | |
| for x in tmodules: | |
| try: | |
| tx = timeit.Timer( | |
| "loads()", | |
| "from shootout import thrash_loads;loads=thrash_loads('%s')" % x) | |
| tx = tx.timeit(number=PERMUTATIONS) | |
| m = sys.modules[x] | |
| print_round(x, tx, to) | |
| except Exception as e: | |
| print "%12s: tests failed: %s" % (x, e) |
Python 3.6.6 on Mac:
..::DUMPS::..
serializer time time-diff size size-diff
json: 4.442723 100.00% 90460 100.00%
simplejson: 4.867349 109.56% 90460 100.00%
ujson: 2.020214 45.47% 89261 98.67%
msgpack: 0.740636 16.67% 86798 95.95%
marshal: 0.623901 14.04% 86909 96.07%
cbor: 1.372539 30.89% 86778 95.93%
pickle: 0.989589 22.27% 89020 98.41%
..::LOADS::..
serializer time time-diff
json: 3.099072 100.00%
simplejson: 2.870310 92.62%
ujson: 3.670731 118.45%
msgpack: 1.002717 32.36%
marshal: 1.046321 33.76%
cbor: 1.803092 58.18%
pickle: 1.298057 41.89%
python3 compatible code is here: https://gist.github.com/crhan/f04ddb7373533bae4d051271497e080e
python 3.6.6 on Mac:
difference:
- use a json data length is: 4978960
PERMUTATIONS = 100
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Python 3.6 on Win7:
Pickle performs really well there.