khmer speed tests

Linux khmer speed test

for threads in {1..12}
do echo Threads: ${threads}
sudo su -c "echo 3 > /proc/sys/vm/drop_caches"
/usr/bin/time load-into-counting.py --threads ${threads} --report-total-kmers null data/25k-casava1_8.fq.bz2
done  |& tee load-log2-readparser

OS X version

for threads in {1..`sysctl -n hw.ncpu`}
do echo Threads: ${threads}
sudo purge
/usr/bin/time load-into-counting.py --threads ${threads} --report-total-kmers null 25k-casava1_8.fq.bz2
done  2>&1 | tee load-log-osx-readparser

and parse with

grep 'user\|Total' load-log-osx-readparser 

linux / khmer v1.2 (ReadParser)

(env)mcrusoe@athyra:~/khmer/gl-master$ grep 'user\|Total' load-log2-readparser                                                                                                                                                                                                            
Total number of unique k-mers: 244751
0.53user 0.08system 0:03.27elapsed 18%CPU (0avgtext+0avgdata 19320maxresident)k
Total number of unique k-mers: 244737
0.70user 0.08system 0:02.63elapsed 29%CPU (0avgtext+0avgdata 19320maxresident)k
Total number of unique k-mers: 244794
0.87user 0.08system 0:03.20elapsed 29%CPU (0avgtext+0avgdata 19256maxresident)k
Total number of unique k-mers: 244821
1.08user 0.08system 0:02.66elapsed 43%CPU (0avgtext+0avgdata 19256maxresident)k
Total number of unique k-mers: 244703
1.51user 0.06system 0:03.11elapsed 50%CPU (0avgtext+0avgdata 21372maxresident)k
Total number of unique k-mers: 244766
2.00user 0.08system 0:02.83elapsed 73%CPU (0avgtext+0avgdata 21372maxresident)k
Total number of unique k-mers: 244803
2.09user 0.04system 0:02.86elapsed 74%CPU (0avgtext+0avgdata 21300maxresident)k
Total number of unique k-mers: 244735
2.42user 0.05system 0:02.78elapsed 89%CPU (0avgtext+0avgdata 21304maxresident)k
Total number of unique k-mers: 244715
2.76user 0.09system 0:02.84elapsed 100%CPU (0avgtext+0avgdata 21368maxresident)k
Total number of unique k-mers: 244783
3.03user 0.06system 0:03.18elapsed 97%CPU (0avgtext+0avgdata 21300maxresident)k
Total number of unique k-mers: 244808
3.83user 0.06system 0:02.99elapsed 130%CPU (0avgtext+0avgdata 21376maxresident)k
Total number of unique k-mers: 244823
3.82user 0.07system 0:02.93elapsed 132%CPU (0avgtext+0avgdata 23672maxresident)k

Linux / Seqan+mutex

(env)mcrusoe@athyra:~/khmer/gl-master$ grep 'user\|Total' load-log2-mutex 
Total number of unique k-mers: 244751
0.64user 0.05system 0:03.95elapsed 17%CPU (0avgtext+0avgdata 19596maxresident)k
Total number of unique k-mers: 244750
0.76user 0.15system 0:02.53elapsed 36%CPU (0avgtext+0avgdata 19620maxresident)k
Total number of unique k-mers: 244751
0.91user 0.18system 0:02.13elapsed 51%CPU (0avgtext+0avgdata 21680maxresident)k
Total number of unique k-mers: 244748
1.09user 0.23system 0:01.94elapsed 68%CPU (0avgtext+0avgdata 19736maxresident)k
Total number of unique k-mers: 244749
1.16user 0.23system 0:02.06elapsed 67%CPU (0avgtext+0avgdata 19752maxresident)k
Total number of unique k-mers: 244750
1.14user 0.25system 0:02.02elapsed 69%CPU (0avgtext+0avgdata 21560maxresident)k
Total number of unique k-mers: 244750
1.17user 0.22system 0:02.21elapsed 63%CPU (0avgtext+0avgdata 21756maxresident)k
Total number of unique k-mers: 244749
1.15user 0.25system 0:02.01elapsed 69%CPU (0avgtext+0avgdata 21720maxresident)k
Total number of unique k-mers: 244751
1.20user 0.21system 0:02.06elapsed 68%CPU (0avgtext+0avgdata 21740maxresident)k
Total number of unique k-mers: 244750
1.17user 0.27system 0:02.12elapsed 68%CPU (0avgtext+0avgdata 21540maxresident)k
Total number of unique k-mers: 244749
1.14user 0.27system 0:01.98elapsed 71%CPU (0avgtext+0avgdata 21724maxresident)k
Total number of unique k-mers: 244750
1.14user 0.25system 0:01.74elapsed 80%CPU (0avgtext+0avgdata 23924maxresident)k

Linux / Seqan+spinlocks

(env)mcrusoe@athyra:~/khmer/gl-master$ grep 'user\|Total' load-log2-spinlocks1                                                                                                                                                                                                          
Total number of unique k-mers: 244751
0.68user 0.06system 0:03.40elapsed 21%CPU (0avgtext+0avgdata 19600maxresident)k
Total number of unique k-mers: 244750
1.26user 0.06system 0:02.40elapsed 55%CPU (0avgtext+0avgdata 19616maxresident)k
Total number of unique k-mers: 244750
1.21user 0.03system 0:01.78elapsed 69%CPU (0avgtext+0avgdata 19720maxresident)k
Total number of unique k-mers: 244747
1.56user 0.04system 0:01.77elapsed 90%CPU (0avgtext+0avgdata 21648maxresident)k
Total number of unique k-mers: 244745
1.71user 0.02system 0:01.61elapsed 107%CPU (0avgtext+0avgdata 19748maxresident)k
Total number of unique k-mers: 244741
2.25user 0.05system 0:01.71elapsed 134%CPU (0avgtext+0avgdata 21568maxresident)k
Total number of unique k-mers: 244740
2.55user 0.05system 0:01.79elapsed 144%CPU (0avgtext+0avgdata 21792maxresident)k
Total number of unique k-mers: 244746
3.34user 0.03system 0:01.83elapsed 184%CPU (0avgtext+0avgdata 19784maxresident)k
Total number of unique k-mers: 244745
4.25user 0.03system 0:02.27elapsed 188%CPU (0avgtext+0avgdata 21752maxresident)k
Total number of unique k-mers: 244745
5.23user 0.04system 0:02.23elapsed 236%CPU (0avgtext+0avgdata 21760maxresident)k
Total number of unique k-mers: 244744
6.46user 0.04system 0:01.82elapsed 357%CPU (0avgtext+0avgdata 23936maxresident)k
Total number of unique k-mers: 244747
8.00user 0.05system 0:02.16elapsed 371%CPU (0avgtext+0avgdata 21740maxresident)k

OS X / ReadParser

ged-iworker:speed-test mcrusoe$ grep 'user\|Total' load-log-osx-readparser 
Total number of unique k-mers: 244751
        2.48 real         0.44 user         0.04 sys
Total number of unique k-mers: 244736
        1.26 real         0.69 user         0.04 sys
Total number of unique k-mers: 244766
        2.17 real         1.07 user         0.04 sys
Total number of unique k-mers: 244756
        1.96 real         1.23 user         0.04 sys

OS X / Seqan+Mutex

ged-iworker:speed-test mcrusoe$ grep 'user\|Total' load-log-osx-seqanmutex 
Total number of unique k-mers: 244751
        5.34 real         0.58 user         0.13 sys
Total number of unique k-mers: 244751
        3.37 real         0.67 user         0.21 sys
Total number of unique k-mers: 244750
        4.80 real         0.69 user         0.25 sys
Total number of unique k-mers: 244751
        6.45 real         0.69 user         0.24 sys

OS X / Seqan+spinlocks

ged-iworker:speed-test mcrusoe$ grep 'user\|Total' load-log-osx-seqan-spin2 
Total number of unique k-mers: 244751
        3.57 real         0.59 user         0.14 sys
Total number of unique k-mers: 244749
        4.16 real         0.79 user         0.14 sys
Total number of unique k-mers: 244746
        4.48 real         1.06 user         0.13 sys
Total number of unique k-mers: 244745
        4.64 real         1.29 user         0.12 sys