martijnvermaat · August 29, 2015 14:07
diff --git a/.gitignore b/.gitignore
 .ipynb_checkpoints
 *.pdf
 *.fa
 [abcde].k1
 [abcde].k2
 [abcde].k3
 [abcde].k4
 [abcde].k5
 [abcde].k6
 [abcde].k7
 [abcde].k8
 [abcde].k9
 [abcde].k10
 [abcde].k11
 [abcde].k12
 [abcde].k13
 [abcde].k14
 [abcde].k15
diff --git a/README.md b/README.md
diff --git a/a.count.k1 b/a.count.k1
 0.542464971542 26120
diff --git a/a.count.k10 b/a.count.k10
 0.663958072662 42492
diff --git a/a.count.k11 b/a.count.k11
 0.972708940506 91608
diff --git a/a.count.k12 b/a.count.k12
 2.24748587608 288264
diff --git a/a.count.k13 b/a.count.k13
 7.51165699959 1074688
diff --git a/a.count.k14 b/a.count.k14
 27.2333760262 4220276
diff --git a/a.count.k15 b/a.count.k15
 108.990545034 16803304
diff --git a/a.count.k2 b/a.count.k2
 0.541092157364 26120
diff --git a/a.count.k3 b/a.count.k3
 0.547954082489 26120
diff --git a/a.count.k4 b/a.count.k4
 0.578961849213 26120
diff --git a/a.count.k5 b/a.count.k5
 0.578741788864 26144
diff --git a/a.count.k6 b/a.count.k6
 0.586236000061 26196
diff --git a/a.count.k7 b/a.count.k7
 0.567816019058 26360
diff --git a/a.count.k8 b/a.count.k8
 0.570757865906 27144
diff --git a/a.count.k9 b/a.count.k9
 0.593155145645 30204
diff --git a/b.count.k1 b/b.count.k1
 5.48644304276 26120
diff --git a/b.count.k10 b/b.count.k10
 5.7480931282 42528
diff --git a/b.count.k11 b/b.count.k11
 6.13077187538 91608
diff --git a/b.count.k12 b/b.count.k12
 7.46066999435 288280
diff --git a/b.count.k13 b/b.count.k13
 13.2450871468 1074684
diff --git a/b.count.k14 b/b.count.k14
 34.4764480591 4220288
diff --git a/b.count.k15 b/b.count.k15
 116.609779835 16803324
diff --git a/b.count.k2 b/b.count.k2
 5.44484806061 26128
diff --git a/b.count.k3 b/b.count.k3
 5.53405499458 26124
diff --git a/b.count.k4 b/b.count.k4
 5.82787203789 26128
diff --git a/b.count.k5 b/b.count.k5
 5.84015107155 26152
diff --git a/b.count.k6 b/b.count.k6
 5.82632112503 26292
diff --git a/b.count.k7 b/b.count.k7
 5.79324197769 26752
diff --git a/b.count.k8 b/b.count.k8
 5.74274420738 27332
diff --git a/b.count.k9 b/b.count.k9
 5.71768307686 30264
diff --git a/c.count.k1 b/c.count.k1
 54.6297440529 26116
diff --git a/c.count.k10 b/c.count.k10
 57.8999540806 43588
diff --git a/c.count.k11 b/c.count.k11
 58.0587861538 91868
diff --git a/c.count.k12 b/c.count.k12
 60.5846529007 288448
diff --git a/c.count.k13 b/c.count.k13
 72.2292790413 1074680
diff --git a/c.count.k14 b/c.count.k14
 93.4170598984 4220460
diff --git a/c.count.k15 b/c.count.k15
 174.824126005 16803412
diff --git a/c.count.k2 b/c.count.k2
 54.5379779339 26120
diff --git a/c.count.k3 b/c.count.k3
 54.4764060974 26120
diff --git a/c.count.k4 b/c.count.k4
 57.6308250427 26124
diff --git a/c.count.k5 b/c.count.k5
 58.1498789787 26144
diff --git a/c.count.k6 b/c.count.k6
 57.4757890701 26264
diff --git a/c.count.k7 b/c.count.k7
 57.5336329937 26756
diff --git a/c.count.k8 b/c.count.k8
 57.6689498425 28424
diff --git a/c.count.k9 b/c.count.k9
 58.0697989464 32832
diff --git a/distance.default.k1 b/distance.default.k1
 0.00376987457275 27948
diff --git a/distance.default.k10 b/distance.default.k10
 0.248427867889 121492
diff --git a/distance.default.k11 b/distance.default.k11
 1.09074687958 397148
diff --git a/distance.default.k12 b/distance.default.k12
 4.51135492325 1499740
diff --git a/distance.default.k13 b/distance.default.k13
 17.9062130451 5669460
diff --git a/distance.default.k14 b/distance.default.k14
 61.405629158 20096264
diff --git a/distance.default.k15 b/distance.default.k15
 199.224109173 63755916
diff --git a/distance.default.k2 b/distance.default.k2
 0.00374603271484 27944
diff --git a/distance.default.k3 b/distance.default.k3
 0.00378704071045 27944
diff --git a/distance.default.k4 b/distance.default.k4
 0.0042028427124 28036
diff --git a/distance.default.k5 b/distance.default.k5
 0.00458002090454 28204
diff --git a/distance.default.k6 b/distance.default.k6
 0.00510907173157 28544
diff --git a/distance.default.k7 b/distance.default.k7
 0.00836801528931 29532
diff --git a/distance.default.k8 b/distance.default.k8
 0.0185878276825 34008
diff --git a/distance.default.k9 b/distance.default.k9
 0.0714981555939 52820
diff --git a/distance.scale.k1 b/distance.scale.k1
 0.00386595726013 27948
diff --git a/distance.scale.k10 b/distance.scale.k10
 0.253719091415 122176
diff --git a/distance.scale.k11 b/distance.scale.k11
 1.1847550869 397036
diff --git a/distance.scale.k12 b/distance.scale.k12
 4.87614607811 1499924
diff --git a/distance.scale.k13 b/distance.scale.k13
 19.3254239559 5670424
diff --git a/distance.scale.k14 b/distance.scale.k14
 68.5020990372 20095424
diff --git a/distance.scale.k15 b/distance.scale.k15
 216.661634207 63754052
diff --git a/distance.scale.k2 b/distance.scale.k2
 0.00380110740662 27936
diff --git a/distance.scale.k3 b/distance.scale.k3
 0.00382900238037 27936
diff --git a/distance.scale.k4 b/distance.scale.k4
 0.00426483154297 28044
diff --git a/distance.scale.k5 b/distance.scale.k5
 0.00455498695374 28208
diff --git a/distance.scale.k6 b/distance.scale.k6
 0.00516986846924 28472
diff --git a/distance.scale.k7 b/distance.scale.k7
 0.00827503204346 29584
diff --git a/distance.scale.k8 b/distance.scale.k8
 0.0180509090424 33936
diff --git a/distance.scale.k9 b/distance.scale.k9
 0.0715341567993 52776
diff --git a/distance.scale.smooth.k1 b/distance.scale.smooth.k1
 0.00382304191589 27940
diff --git a/distance.scale.smooth.k10 b/distance.scale.smooth.k10
 11.709302187 122136
diff --git a/distance.scale.smooth.k11 b/distance.scale.smooth.k11
 46.8763818741 392168
diff --git a/distance.scale.smooth.k12 b/distance.scale.smooth.k12
 191.472046852 1367092
diff --git a/distance.scale.smooth.k13 b/distance.scale.smooth.k13
 694.941875935 4249648
diff --git a/distance.scale.smooth.k14 b/distance.scale.smooth.k14
 1946.50338984 13000560
diff --git a/distance.scale.smooth.k15 b/distance.scale.smooth.k15
 5011.18195319 42178040
diff --git a/distance.scale.smooth.k2 b/distance.scale.smooth.k2
 0.00398802757263 27940
diff --git a/distance.scale.smooth.k3 b/distance.scale.smooth.k3
 0.00459003448486 27940
diff --git a/distance.scale.smooth.k4 b/distance.scale.smooth.k4
 0.00705599784851 27952
diff --git a/distance.scale.smooth.k5 b/distance.scale.smooth.k5
 0.0167629718781 28212
diff --git a/distance.scale.smooth.k6 b/distance.scale.smooth.k6
 0.0539407730103 28568
diff --git a/distance.scale.smooth.k7 b/distance.scale.smooth.k7
 0.206032991409 29584
diff --git a/distance.scale.smooth.k8 b/distance.scale.smooth.k8
 0.806213140488 33940
diff --git a/distance.scale.smooth.k9 b/distance.scale.smooth.k9
 3.21424412727 52784
diff --git a/distance.smooth.k1 b/distance.smooth.k1
 0.0038058757782 27944
diff --git a/distance.smooth.k10 b/distance.smooth.k10
 11.3999941349 121544
diff --git a/distance.smooth.k11 b/distance.smooth.k11
 44.8743479252 392128
diff --git a/distance.smooth.k12 b/distance.smooth.k12
 209.745757103 1367460
diff --git a/distance.smooth.k13 b/distance.smooth.k13
 775.451279879 4249036
diff --git a/distance.smooth.k14 b/distance.smooth.k14
 2038.04450011 13000368
diff --git a/distance.smooth.k15 b/distance.smooth.k15
 4487.34653401 39471116
diff --git a/distance.smooth.k2 b/distance.smooth.k2
 0.00396203994751 27940
diff --git a/distance.smooth.k3 b/distance.smooth.k3
 0.00455594062805 27944
diff --git a/distance.smooth.k4 b/distance.smooth.k4
 0.00734496116638 28048
diff --git a/distance.smooth.k5 b/distance.smooth.k5
 0.0167670249939 28208
diff --git a/distance.smooth.k6 b/distance.smooth.k6
 0.0530319213867 28560
diff --git a/distance.smooth.k7 b/distance.smooth.k7
 0.208234071732 29528
diff --git a/distance.smooth.k8 b/distance.smooth.k8
 0.775527000427 34012
diff --git a/distance.smooth.k9 b/distance.smooth.k9
 3.10026812553 52828
diff --git a/profile_count.py b/profile_count.py
 #!/usr/bin/env python

 import resource
 import sys
 import timeit

 from k_mer import klib


 def index(filename, k):
    with open(filename) as f:
        profile = klib.Profile.from_fasta(f, k)


 def profile_index(filename, k):
    repeats = timeit.repeat('index({0}, {1})'.format(repr(filename), k),
                            setup='from __main__ import index',
                            repeat=3, number=1)
    usage_time = min(repeats)
    usage_memory = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss

    return usage_time, usage_memory


 if __name__ == '__main__':
    if len(sys.argv) != 3:
        sys.stdout.write('Usage: %s SAMPLE K\n' % sys.argv[0])
        sys.exit(1)
    usage_time, usage_memory = profile_index(sys.argv[1], int(sys.argv[2]))
    print usage_time, usage_memory
diff --git a/profile_count.sh b/profile_count.sh
 #!/bin/bash

 for fasta in a.fa b.fa c.fa; do
    # Copy it to local storage
    local_fasta=$(mktemp)
    cp $fasta $local_fasta

    # Read it once, so we have no cache effects
    ./profile_count.py $local_fasta 4 > /dev/null

    for k in {1..15}; do
        ./profile_count.py $local_fasta $k > ${fasta%.fa}.count.k${k}
    done

    rm $local_fasta
 done
diff --git a/profile_distance.py b/profile_distance.py
 #!/usr/bin/env python


 import resource
 import sys
 import timeit

 from k_mer import ProfileFileType, kdistlib, klib


 def distance(filename_a, filename_b, **diff_args):
    with ProfileFileType()(filename_a) as f_a, ProfileFileType()(filename_b) as f_b:
        profile_a = klib.Profile.from_file(f_a)
        profile_b = klib.Profile.from_file(f_b)

        if profile_a.length != profile_b.length:
            raise ValueError('trying to compare profiles for different k')

        dist = kdistlib.ProfileDistance(**diff_args)
        dist.distance(profile_a, profile_b)


 def profile_distance(filename_a, filename_b, **diff_args):
    repeats = timeit.repeat('distance({0}, {1}, **{2})'.format(repr(filename_a),
                                                               repr(filename_b),
                                                               repr(diff_args)),
                            setup='from __main__ import distance',
                            repeat=3, number=1)
    usage_time = min(repeats)
    usage_memory = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss

    return usage_time, usage_memory


 if __name__ == '__main__':
    if len(sys.argv) != 4:
        sys.stdout.write('Usage: %s PROFILE_LEFT PROFILE_RIGHT MODE\n' % sys.argv[0])
        sys.exit(1)
    if sys.argv[3] not in ('default', 'scale', 'smooth', 'scale.smooth'):
        sys.stdout.write('Not a valid mode: %s\n' % sys.argv[3])
        sys.exit(1)

    if sys.argv[3] == 'default':
        usage_time, usage_memory = profile_distance(sys.argv[1], sys.argv[2])
    elif sys.argv[3] == 'scale':
        usage_time, usage_memory = profile_distance(sys.argv[1], sys.argv[2], do_scale=True)
    elif sys.argv[3] == 'smooth':
        usage_time, usage_memory = profile_distance(sys.argv[1], sys.argv[2], do_smooth=True, threshold=10)
    elif sys.argv[3] == 'scale.smooth':
        usage_time, usage_memory = profile_distance(sys.argv[1], sys.argv[2], do_scale=True, do_smooth=True, threshold=10)

    print usage_time, usage_memory
diff --git a/profile_distance.sh b/profile_distance.sh
 #!/bin/bash

 # Copy fasta files to local storage to speed things up
 local_fasta_d=$(mktemp)
 local_fasta_e=$(mktemp)
 cp d.fa $local_fasta_d
 cp e.fa $local_fasta_e

 for k in {1..15}; do
    # Create profiles
    kMer count -k $k d.k${k} $local_fasta_d
    kMer count -k $k e.k${k} $local_fasta_e

    # Copy profiles to local storage
    local_d=$(mktemp)
    local_e=$(mktemp)
    cp d.k${k} $local_d
    cp e.k${k} $local_e

    # Read them once, so we have no cache effects
    kMer info $local_d > /dev/null
    kMer info $local_e > /dev/null

    ./profile_distance.py $local_d $local_e default > distance.default.k${k}
    ./profile_distance.py $local_d $local_e scale > distance.scale.k${k}
    ./profile_distance.py $local_d $local_e smooth > distance.smooth.k${k}
    ./profile_distance.py $local_d $local_e scale.smooth > distance.scale.smooth.k${k}

    rm $local_d $local_e
 done

 rm $local_fasta_d $local_fasta_e
diff --git a/profiling.ipynb b/profiling.ipynb
	.ipynb_checkpoints
	*.pdf
	*.fa
	[abcde].k1
	[abcde].k2
	[abcde].k3
	[abcde].k4
	[abcde].k5
	[abcde].k6
	[abcde].k7
	[abcde].k8
	[abcde].k9
	[abcde].k10
	[abcde].k11
	[abcde].k12
	[abcde].k13
	[abcde].k14
	[abcde].k15
	#!/usr/bin/env python

	import resource
	import sys
	import timeit

	from k_mer import klib


	def index(filename, k):
	with open(filename) as f:
	profile = klib.Profile.from_fasta(f, k)


	def profile_index(filename, k):
	repeats = timeit.repeat('index({0}, {1})'.format(repr(filename), k),
	setup='from __main__ import index',
	repeat=3, number=1)
	usage_time = min(repeats)
	usage_memory = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss

	return usage_time, usage_memory


	if __name__ == '__main__':
	if len(sys.argv) != 3:
	sys.stdout.write('Usage: %s SAMPLE K\n' % sys.argv[0])
	sys.exit(1)
	usage_time, usage_memory = profile_index(sys.argv[1], int(sys.argv[2]))
	print usage_time, usage_memory
	#!/bin/bash

	for fasta in a.fa b.fa c.fa; do
	# Copy it to local storage
	local_fasta=$(mktemp)
	cp $fasta $local_fasta

	# Read it once, so we have no cache effects
	./profile_count.py $local_fasta 4 > /dev/null

	for k in {1..15}; do
	./profile_count.py $local_fasta $k > ${fasta%.fa}.count.k${k}
	done

	rm $local_fasta
	done
	#!/bin/bash

	# Copy fasta files to local storage to speed things up
	local_fasta_d=$(mktemp)
	local_fasta_e=$(mktemp)
	cp d.fa $local_fasta_d
	cp e.fa $local_fasta_e

	for k in {1..15}; do
	# Create profiles
	kMer count -k $k d.k${k} $local_fasta_d
	kMer count -k $k e.k${k} $local_fasta_e

	# Copy profiles to local storage
	local_d=$(mktemp)
	local_e=$(mktemp)
	cp d.k${k} $local_d
	cp e.k${k} $local_e

	# Read them once, so we have no cache effects
	kMer info $local_d > /dev/null
	kMer info $local_e > /dev/null

	./profile_distance.py $local_d $local_e default > distance.default.k${k}
	./profile_distance.py $local_d $local_e scale > distance.scale.k${k}
	./profile_distance.py $local_d $local_e smooth > distance.smooth.k${k}
	./profile_distance.py $local_d $local_e scale.smooth > distance.scale.smooth.k${k}

	rm $local_d $local_e
	done

	rm $local_fasta_d $local_fasta_e