Created
February 1, 2019 20:24
-
-
Save pepasflo/9babaf9a3d955229a315663a7967b355 to your computer and use it in GitHub Desktop.
speed comparison of md5 and crc32 (python)
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
jpepas@radium$ cat do-md5.py | |
#!/usr/bin/env python | |
import sys | |
import hashlib | |
h = hashlib.md5() | |
with open(sys.argv[1], 'rb') as f: | |
# thanks to https://stackoverflow.com/a/3431838/7543271 | |
for chunk in iter(lambda: f.read(4096 * 64), b""): | |
h.update(chunk) | |
print h.hexdigest() | |
jpepas@radium$ cat do-crc.py | |
#!/usr/bin/env python | |
import sys | |
import zlib | |
checksum = zlib.crc32(open(sys.argv[1], 'rb').read()) | |
print "%x" % (checksum & 0xFFFFFFFF) | |
jpepas@radium$ cat do-crc2.py | |
#!/usr/bin/env python | |
import sys | |
import zlib | |
with open(sys.argv[1], 'rb') as f: | |
# thanks to https://stackoverflow.com/a/3431838/7543271 | |
# thanks to https://wiki-bsse.ethz.ch/display/DBSSEQGF/Calculating+CRC32+checksum+with+Python | |
accumulator = 0 | |
for chunk in iter(lambda: f.read(4096 * 64), b""): | |
accumulator = zlib.crc32(chunk, accumulator) | |
print "%x" % (accumulator & 0xFFFFFFFF) | |
jpepas@radium$ dd if=/dev/urandom of=random.bin bs=1m count=100 | |
100+0 records in | |
100+0 records out | |
104857600 bytes transferred in 8.487965 secs (12353679 bytes/sec) | |
jpepas@radium$ export TIMEFORMAT='%3R' | |
jpepas@radium$ for i in 1 2 3; do time ./do-md5.py random.bin > /dev/null; done | |
1.159 | |
1.169 | |
1.170 | |
jpepas@radium$ for i in 1 2 3; do time ./do-crc.py random.bin > /dev/null; done | |
0.151 | |
0.151 | |
0.134 | |
jpepas@radium$ for i in 1 2 3; do time ./do-crc2.py random.bin > /dev/null; done | |
0.078 | |
0.070 | |
0.067 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment