Skip to content

Instantly share code, notes, and snippets.

@jvhaarst

jvhaarst/Results.tsv

Created April 28, 2011 09:57
Show Gist options
  • Save jvhaarst/946108 to your computer and use it in GitHub Desktop.
Save jvhaarst/946108 to your computer and use it in GitHub Desktop.
Download ZIP
Script to benchmark compression and decompression of a Velvet Sequences file
Raw
compresstest.sh
#!/bin/bash
#pbzip2 : http://www.compression.ca/pbzip2/
#pigz : http://www.zlib.net/pigz/
#pixz : https://github.com/vasi/pixz (aptitude install liblzma-dev libarchive-dev && make)
#tamp : http://blogs.sun.com/timc/entry/tamp_a_lightweight_multi_threaded
#p7zip : http://sourceforge.net/projects/p7zip/files/p7zip/
#lzop : http://www.lzop.org/
START=1
END=9
NAME=test.rawImages.tar # d0eed0574ad223ff2b43448903de61f8
NAME=Sequences # 70f61af7d55e3306cefeb36cc4fb04fe
export TMPDIR=`pwd`
echo gz
for block in `seq 1 ${END}`
do
echo "$block"
/usr/bin/time --format "%e\t%S\t%U\t%P" pigz -${block} -k --suffix .${block}.gz ${NAME}
FILE=`mktemp -u tmp.XXXXXXX.gz`
/usr/bin/time --format "%e\t%S\t%U\t%P" pigz -d -c ${NAME}.${block}.gz > $FILE
stat --printf="%s\t" ${NAME} $FILE ${NAME}.${block}.gz
echo
md5sum $FILE | cut -f 1 -d' '
rm $FILE
rm ${NAME}.${block}.gz
done
echo bzip2
for block in `seq 1 ${END}`
do
echo $block
/usr/bin/time --format "%e\t%S\t%U\t%P" pbzip2 -${block} -k ${NAME} -c > ${NAME}.${block}.bz2
FILE=`mktemp -u tmp.XXXXXXX.bz2`
/usr/bin/time --format "%e\t%S\t%U\t%P" pbzip2 -d -c ${NAME}.${block}.bz2 > $FILE
stat --printf="%s\t" ${NAME} $FILE ${NAME}.${block}.bz2
echo
md5sum $FILE | cut -f 1 -d' '
rm $FILE
rm ${NAME}.${block}.bz2
done
echo xz
for block in `seq 1 ${END}`
do
echo $block
/usr/bin/time --format "%e\t%S\t%U\t%P" ~/code/pixz/pixz -${block} -i ${NAME} -o ${NAME}.${block}.xz
FILE=`mktemp -u tmp.XXXXXXX.xz`
/usr/bin/time --format "%e\t%S\t%U\t%P" ~/code/pixz/pixz -x -i ${NAME}.${block}.xz -o $FILE
stat --printf="%s\t" ${NAME} $FILE ${NAME}.${block}.xz
echo
md5sum $FILE | cut -f 1 -d' '
rm $FILE
rm ${NAME}.${block}.xz
done
echo 7zip-lzma
for block in `seq 1 ${END}`
do
echo $block
/usr/bin/time --format "%e\t%S\t%U\t%P" /home/jvh/effe/p7zip_9.20.1/bin/7zr a -bd -t7z -mmt=on -mx=${block} -m0=lzma ${NAME}.${block}.7z ${NAME}
FILE=`mktemp -u tmp.XXXXXXX.7z`
/usr/bin/time --format "%e\t%S\t%U\t%P" /home/jvh/effe/p7zip_9.20.1/bin/7zr e -bd -so -mmt=on ${NAME}.${block}.7z > $FILE
stat --printf="%s\t" ${NAME} $FILE ${NAME}.${block}.7z
echo
md5sum $FILE | cut -f 1 -d' '
rm $FILE
rm ${NAME}.${block}.7z
done
echo 7zip-lzma2
for block in `seq 1 ${END}`
do
echo $block
/usr/bin/time --format "%e\t%S\t%U\t%P" /home/jvh/effe/p7zip_9.20.1/bin/7zr a -bd -t7z -mmt=on -mx=${block} -m0=lzma2 ${NAME}.${block}.7z ${NAME}
FILE=`mktemp -u tmp.XXXXXXX.7z2`
/usr/bin/time --format "%e\t%S\t%U\t%P" /home/jvh/effe/p7zip_9.20.1/bin/7zr e -bd -so -mmt=on ${NAME}.${block}.7z > $FILE
stat --printf="%s\t" ${NAME} $FILE ${NAME}.${block}.7z
echo
md5sum $FILE | cut -f 1 -d' '
rm $FILE
rm ${NAME}.${block}.7z
done
echo zip-deflate
for block in `seq 1 ${END}`
do
echo $block
/usr/bin/time --format "%e\t%S\t%U\t%P" zip -${block} --quiet ${NAME}.${block}.zip ${NAME}
FILE=`mktemp -u tmp.XXXXXXX.zip`
/usr/bin/time --format "%e\t%S\t%U\t%P" unzip -p ${NAME}.${block}.zip > $FILE
stat --printf="%s\t" ${NAME} $FILE ${NAME}.${block}.zip
echo
md5sum $FILE | cut -f 1 -d' '
rm $FILE
rm ${NAME}.${block}.zip
done
echo tamp
for block in 1
do
echo $block
/usr/bin/time --format "%e\t%S\t%U\t%P" /home/jvh/code/tamp/tamp-2.5/tamp -i ${NAME} -o ${NAME}.${block}.q
FILE=`mktemp -u tmp.XXXXXXX.tamp`
/usr/bin/time --format "%e\t%S\t%U\t%P" /home/jvh/code/tamp/tamp-2.5/tamp -d -i ${NAME}.${block}.q -o $FILE
stat --printf="%s\t" ${NAME} $FILE ${NAME}.${block}.q
echo
md5sum $FILE | cut -f 1 -d' '
rm $FILE
rm ${NAME}.${block}.q
done
echo lzop
for block in `seq 1 ${END}`
do
echo "$block"
/usr/bin/time --format "%e\t%S\t%U\t%P" lzop -${block} -k -c ${NAME} > ${NAME}.${block}.lz
FILE=`mktemp -u tmp.XXXXXXX.lzop`
/usr/bin/time --format "%e\t%S\t%U\t%P" lzop -d -c ${NAME}.${block}.lz > $FILE
stat --printf="%s\t" ${NAME} $FILE ${NAME}.${block}.lz
echo
md5sum $FILE | cut -f 1 -d' '
rm $FILE
rm ${NAME}.${block}.lz
done
# Time format used:
# e Elapsed real (wall clock) time used by the process, in seconds.
# S Total number of CPU-seconds used by the system on behalf of the process (in kernel mode), in seconds.
# U Total number of CPU-seconds that the process used directly (in user mode), in seconds.
# P Percentage of the CPU that this job got. This is just user + system times divided by the total running time. It also prints a percentage sign.
Raw
Results.tsv
Compression Decompression
Type Setting Wall clock time System time User time CPU Wall clock time System time User time CPU Original size Uncompressed size Compressed sixe Percentage of original size
gz 1 3.41 1.4 63.53 1904% 6.5 1.49 8.61 155% 1382185155 1382185155 412365877 30%
gz 2 3.37 1.17 63.77 1921% 6.2 1.6 8.39 161% 1382185155 1382185155 396821613 29%
gz 3 5.19 1.4 96.58 1885% 5.78 1.41 7.74 158% 1382185155 1382185155 382874207 28%
gz 4 3.76 1.21 67.11 1812% 5.56 1.58 7.42 161% 1382185155 1382185155 372277033 27%
gz 5 7.48 1.65 141.5 1911% 5.35 1.76 7.13 166% 1382185155 1382185155 363078787 26%
gz 6 11.57 1.21 237.77 2063% 5.28 1.78 7.18 169% 1382185155 1382185155 352875787 26%
gz 7 21.74 1.32 453.06 2089% 5.3 1.4 7.34 164% 1382185155 1382185155 344893059 25%
gz 8 30.57 1.4 659.53 2161% 5.33 1.23 7.32 160% 1382185155 1382185155 343665387 25%
gz 9 33.13 1.6 743.44 2248% 5.27 1.49 7.62 172% 1382185155 1382185155 343665412 25%
bzip2 1 9.23 1.04 206.64 2249% 3.04 1.57 66.24 2225% 1382185155 1382185155 296543842 21%
bzip2 2 9.32 1.55 207.68 2244% 3.33 1.73 71.1 2181% 1382185155 1382185155 296684036 21%
bzip2 3 9.67 1.45 214.39 2231% 3.61 1.84 74.21 2101% 1382185155 1382185155 297885403 22%
bzip2 4 10.19 1.69 218.66 2161% 3.75 1.7 78.88 2143% 1382185155 1382185155 299022346 22%
bzip2 5 10.37 1.53 221.63 2151% 3.95 1.69 82.03 2117% 1382185155 1382185155 300326711 22%
bzip2 6 10.33 1.56 222.41 2167% 4.02 2.05 83.7 2131% 1382185155 1382185155 300869311 22%
bzip2 7 10.33 1.72 229.34 2236% 4.16 1.91 89.76 2202% 1382185155 1382185155 301886696 22%
bzip2 8 10.67 1.82 238.4 2249% 4.48 1.81 98 2223% 1382185155 1382185155 303563160 22%
bzip2 9 12.17 2.3 271.8 2252% 5.1 2.04 108.27 2159% 1382185155 1382185155 305918500 22%
xz 1 10.74 1.45 223.79 2095% 2.29 1.66 45.23 2040% 1382185155 1382185155 335965340 24%
xz 2 10.12 1.37 226.69 2251% 2.13 1.61 41.33 2007% 1382185155 1382185155 323172572 23%
xz 3 61.37 2.21 1353.84 2209% 1.97 1.57 40.72 2145% 1382185155 1382185155 263421660 19%
xz 4 73.34 3.55 1649.74 2254% 2.2 1.75 45.88 2156% 1382185155 1382185155 257518940 19%
xz 5 93.62 5.18 2020.81 2164% 1.98 1.62 37.14 1952% 1382185155 1382185155 250521768 18%
xz 6 123.52 6.59 2623.35 2129% 2.36 2.13 42.35 1883% 1382185155 1382185155 241273104 17%
xz 7 151.62 12.58 3157.27 2090% 2.27 2.55 35.05 1656% 1382185155 1382185155 227389108 16%
xz 8 177.12 17.56 3407 1933% 2.38 2.66 34.08 1537% 1382185155 1382185155 208039020 15%
xz 9 191.71 18.26 3542.85 1857% 2.87 2.4 29.17 1098% 1382185155 1382185155 186511124 13%
7zip-lzma 1 164.76 0.56 163.86 99% 30.8 0.95 29.79 99% 1382185155 1382185155 321271225 23%
7zip-lzma 2 166.82 0.57 165.92 99% 30.83 0.82 29.95 99% 1382185155 1382185155 321271225 23%
7zip-lzma 3 167.44 0.5 166.61 99% 32.31 0.9 31.33 99% 1382185155 1382185155 322047925 23%
7zip-lzma 4 166.62 0.54 165.75 99% 30.59 0.78 29.76 99% 1382185155 1382185155 322047925 23%
7zip-lzma 5 1084.5 4.32 1871.26 172% 22.82 0.92 21.85 99% 1382185155 1382185155 206500051 15%
7zip-lzma 6 1220.31 4.38 1901.89 156% 24.02 1.08 22.9 99% 1382185155 1382185155 206500051 15%
7zip-lzma 7 1540.33 5.57 2245.05 146% 23.63 1.09 22.5 99% 1382185155 1382185155 180096517 13%
7zip-lzma 8 1616.42 4.91 2347.08 145% 23.91 0.86 23.01 99% 1382185155 1382185155 180096517 13%
7zip-lzma 9 1569.97 5.25 2395.22 152% 21.99 0.86 21.08 99% 1382185155 1382185155 158207608 11%
7zip-lzma2 1 16.47 1.08 331.35 2017% 31.82 0.88 30.88 99% 1382185155 1382185155 322008830 23%
7zip-lzma2 2 16.19 1.16 320.7 1987% 31.79 1.08 30.65 99% 1382185155 1382185155 322008830 23%
7zip-lzma2 3 14.78 1.09 287.11 1948% 31.37 0.73 30.58 99% 1382185155 1382185155 322257450 23%
7zip-lzma2 4 14.23 1.17 278.42 1963% 30.85 0.82 29.97 99% 1382185155 1382185155 322257450 23%
7zip-lzma2 5 194.45 9.28 2984.95 1539% 25.39 1.09 24.26 99% 1382185155 1382185155 209188035 15%
7zip-lzma2 6 192.62 8.99 2854.4 1486% 24.41 0.84 23.53 99% 1382185155 1382185155 209188035 15%
7zip-lzma2 7
7zip-lzma2 8
7zip-lzma2 9
zip-deflate 1 27.1 0.67 26.37 99% 13.29 0.92 12.34 99% 1382185155 1382185155 411870251 30%
zip-deflate 2 32.34 0.64 31.63 99% 13.18 0.77 12.39 99% 1382185155 1382185155 396303744 29%
zip-deflate 3 51.79 0.64 51.05 99% 11.47 0.69 10.76 99% 1382185155 1382185155 382506213 28%
zip-deflate 4 42.14 0.51 41.53 99% 11.62 0.8 10.8 99% 1382185155 1382185155 371786757 27%
zip-deflate 5 86.5 0.63 85.7 99% 11.31 0.72 10.57 99% 1382185155 1382185155 362475152 26%
zip-deflate 6 213.54 0.58 212.54 99% 10.68 0.87 9.79 99% 1382185155 1382185155 352266277 25%
zip-deflate 7 355.42 0.83 353.9 99% 11.11 0.95 10.14 99% 1382185155 1382185155 344298523 25%
zip-deflate 8 519.58 0.76 517.8 99% 10.26 1.01 9.23 99% 1382185155 1382185155 343075056 25%
zip-deflate 9 514.85 0.66 513.19 99% 10.78 0.77 9.99 99% 1382185155 1382185155 343075068 25%
tamp 1 1.24 1.26 8.9 815% 3.04 2.11 8.68 354% 1382185155 1382185155 640657770 46%
lzop 1 7.41 0.73 6.67 99% 5.42 0.96 4.45 99% 1382185155 1382185155 584286704 42%
lzop 2 6.99 0.54 6.44 99% 5.44 1 4.43 99% 1382185155 1382185155 584554496 42%
lzop 3 7.18 0.63 6.54 99% 5.34 0.88 4.45 99% 1382185155 1382185155 584554496 42%
lzop 4 7.32 0.69 6.62 99% 5.46 0.8 4.64 99% 1382185155 1382185155 584554496 42%
lzop 5 6.99 0.61 6.37 99% 5.55 1.05 4.49 99% 1382185155 1382185155 584554496 42%
lzop 6 7.05 0.62 6.41 99% 5.29 0.97 4.31 99% 1382185155 1382185155 584554496 42%
lzop 7 407.5 0.58 406.1 99% 4.87 0.92 3.9 98% 1382185155 1382185155 446427173 32%
lzop 8 866.12 0.56 863.88 99% 4.89 0.78 4.1 99% 1382185155 1382185155 433429451 31%
lzop 9 858.5 0.51 856.32 99% 5.23 1.01 4.21 99% 1382185155 1382185155 433429440 31%
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment