pybedtools manuscript

performance-comparo.sh

export PBTHOME=/home/arq5x/cphg-home/projects/pybedtools

shuffle 1094PC0012.bed > a.shuf.full.bed
shuffle 1094PC0012.bed > b.shuf.full.bed
head -1000 a.shuf.full.bed > a.shuf.1e3.bed; head -1000 b.shuf.full.bed > b.shuf.1e3.bed
head -10000 a.shuf.full.bed > a.shuf.1e4.bed; head -10000 b.shuf.full.bed > b.shuf.1e4.bed
head -100000 a.shuf.full.bed > a.shuf.1e5.bed; head -100000 b.shuf.full.bed > b.shuf.1e5.bed
head -1000000 a.shuf.full.bed > a.shuf.1e6.bed; head -1000000 b.shuf.full.bed > b.shuf.1e6.bed
head -10000000 a.shuf.full.bed > a.shuf.1e7.bed; head -10000000 b.shuf.full.bed > b.shuf.1e7.bed
head -100000000 a.shuf.full.bed > a.shuf.1e8.bed; head -100000000 b.shuf.full.bed > b.shuf.1e8.bed

###################################################
# intersectBed full join (-wa, -wb)
###################################################
echo "cd $PBTHOME; runit intersectBed -a a.shuf.1e3.bed -b b.shuf.1e3.bed -wa -wb > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=2000m:ncpus=1 -m bea -M [email protected] -N bt-e3-join
echo "cd $PBTHOME; runit intersectBed -a a.shuf.1e4.bed -b b.shuf.1e4.bed -wa -wb > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=4000m:ncpus=1 -m bea -M [email protected] -N bt-e4-join
echo "cd $PBTHOME; runit intersectBed -a a.shuf.1e5.bed -b b.shuf.1e5.bed -wa -wb > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=8000m:ncpus=1 -m bea -M [email protected] -N bt-e5-join
echo "cd $PBTHOME; runit intersectBed -a a.shuf.1e6.bed -b b.shuf.1e6.bed -wa -wb > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=16000m:ncpus=1 -m bea -M [email protected] -N bt-e6-join
echo "cd $PBTHOME; runit intersectBed -a a.shuf.1e7.bed -b b.shuf.1e7.bed -wa -wb > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=64000m:ncpus=1 -m bea -M [email protected] -N bt-e7-join
echo "cd $PBTHOME; runit intersectBed -a a.shuf.1e8.bed -b b.shuf.1e8.bed -wa -wb > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=64000m:ncpus=1 -m bea -M [email protected] -N bt-e8-join

###################################################
# intersectBed full join (-u)
###################################################
echo "cd $PBTHOME; runit intersectBed -a a.shuf.1e3.bed -b b.shuf.1e3.bed -u > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=2000m:ncpus=1 -m bea -M [email protected] -N bt-e3-uniq
echo "cd $PBTHOME; runit intersectBed -a a.shuf.1e4.bed -b b.shuf.1e4.bed -u > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=4000m:ncpus=1 -m bea -M [email protected] -N bt-e4-uniq
echo "cd $PBTHOME; runit intersectBed -a a.shuf.1e5.bed -b b.shuf.1e5.bed -u > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=8000m:ncpus=1 -m bea -M [email protected] -N bt-e5-uniq
echo "cd $PBTHOME; runit intersectBed -a a.shuf.1e6.bed -b b.shuf.1e6.bed -u > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=16000m:ncpus=1 -m bea -M [email protected] -N bt-e6-uniq
echo "cd $PBTHOME; runit intersectBed -a a.shuf.1e7.bed -b b.shuf.1e7.bed -u > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=64000m:ncpus=1 -m bea -M [email protected] -N bt-e7-uniq
echo "cd $PBTHOME; runit intersectBed -a a.shuf.1e8.bed -b b.shuf.1e8.bed -u > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=64000m:ncpus=1 -m bea -M [email protected] -N bt-e8-uniq

###################################################
# bedIntersect full join* not really
###################################################
echo "cd $PBTHOME; runit bedIntersect a.shuf.1e3.bed b.shuf.1e3.bed stdout > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=2000m:ncpus=1 -m bea -M [email protected] -N kent-e3-join
echo "cd $PBTHOME; runit bedIntersect a.shuf.1e4.bed b.shuf.1e4.bed stdout > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=4000m:ncpus=1 -m bea -M [email protected] -N kent-e4-join
echo "cd $PBTHOME; runit bedIntersect a.shuf.1e5.bed b.shuf.1e5.bed stdout > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=8000m:ncpus=1 -m bea -M [email protected] -N kent-e5-join
echo "cd $PBTHOME; runit bedIntersect a.shuf.1e6.bed b.shuf.1e6.bed stdout > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=16000m:ncpus=1 -m bea -M [email protected] -N kent-e6-join
echo "cd $PBTHOME; runit bedIntersect a.shuf.1e7.bed b.shuf.1e7.bed stdout > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=64000m:ncpus=1 -m bea -M [email protected] -N kent-e7-join
echo "cd $PBTHOME; runit bedIntersect a.shuf.1e8.bed b.shuf.1e8.bed stdout > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=64000m:ncpus=1 -m bea -M [email protected] -N kent-e8-join

###################################################
# bedIntersect uniq
###################################################
echo "cd $PBTHOME; runit bedIntersect -aHitAny a.shuf.1e3.bed b.shuf.1e3.bed stdout > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=2000m:ncpus=1 -m bea -M [email protected] -N kent-e3-uniq
echo "cd $PBTHOME; runit bedIntersect -aHitAny a.shuf.1e4.bed b.shuf.1e4.bed stdout > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=4000m:ncpus=1 -m bea -M [email protected] -N kent-e4-uniq
echo "cd $PBTHOME; runit bedIntersect -aHitAny a.shuf.1e5.bed b.shuf.1e5.bed stdout > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=8000m:ncpus=1 -m bea -M [email protected] -N kent-e5-uniq
echo "cd $PBTHOME; runit bedIntersect -aHitAny a.shuf.1e6.bed b.shuf.1e6.bed stdout > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=16000m:ncpus=1 -m bea -M [email protected] -N kent-e6-uniq
echo "cd $PBTHOME; runit bedIntersect -aHitAny a.shuf.1e7.bed b.shuf.1e7.bed stdout > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=64000m:ncpus=1 -m bea -M [email protected] -N kent-e7-uniq
echo "cd $PBTHOME; runit bedIntersect -aHitAny a.shuf.1e8.bed b.shuf.1e8.bed stdout > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=64000m:ncpus=1 -m bea -M [email protected] -N kent-e8-uniq

####################################################
# pybedtools full
####################################################
echo "cd $PBTHOME; runit ./pybedtools-intersect-full.py a.shuf.1e3.bed b.shuf.1e3.bed > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=2000m:ncpus=1 -m bea -M [email protected] -N pbt-e3-full
echo "cd $PBTHOME; runit ./pybedtools-intersect-full.py a.shuf.1e4.bed b.shuf.1e4.bed > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=4000m:ncpus=1 -m bea -M [email protected] -N pbt-e4-full
echo "cd $PBTHOME; runit ./pybedtools-intersect-full.py a.shuf.1e5.bed b.shuf.1e5.bed > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=8000m:ncpus=1 -m bea -M [email protected] -N pbt-e5-full
echo "cd $PBTHOME; runit ./pybedtools-intersect-full.py a.shuf.1e6.bed b.shuf.1e6.bed > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=16000m:ncpus=1 -m bea -M [email protected] -N pbt-e6-full
echo "cd $PBTHOME; runit ./pybedtools-intersect-full.py a.shuf.1e7.bed b.shuf.1e7.bed > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=64000m:ncpus=1 -m bea -M [email protected] -N pbt-e7-full
echo "cd $PBTHOME; runit ./pybedtools-intersect-full.py a.shuf.1e8.bed b.shuf.1e8.bed > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=64000m:ncpus=1 -m bea -M [email protected] -N pbt-e8-full

####################################################
# pybedtools uniq
####################################################
echo "cd $PBTHOME; runit ./pybedtools-intersect-uniq.py a.shuf.1e3.bed b.shuf.1e3.bed > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=2000m:ncpus=1 -m bea -M [email protected] -N pbt-e3-uniq
echo "cd $PBTHOME; runit ./pybedtools-intersect-uniq.py a.shuf.1e4.bed b.shuf.1e4.bed > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=4000m:ncpus=1 -m bea -M [email protected] -N pbt-e4-uniq
echo "cd $PBTHOME; runit ./pybedtools-intersect-uniq.py a.shuf.1e5.bed b.shuf.1e5.bed > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=8000m:ncpus=1 -m bea -M [email protected] -N pbt-e5-uniq
echo "cd $PBTHOME; runit ./pybedtools-intersect-uniq.py a.shuf.1e6.bed b.shuf.1e6.bed > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=16000m:ncpus=1 -m bea -M [email protected] -N pbt-e6-uniq
echo "cd $PBTHOME; runit ./pybedtools-intersect-uniq.py a.shuf.1e7.bed b.shuf.1e7.bed > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=64000m:ncpus=1 -m bea -M [email protected] -N pbt-e7-uniq
echo "cd $PBTHOME; runit ./pybedtools-intersect-uniq.py a.shuf.1e8.bed b.shuf.1e8.bed > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=64000m:ncpus=1 -m bea -M [email protected] -N pbt-e8-uniq

####################################################
# bx-python full
####################################################
echo "cd $PBTHOME; runit interval_join.py a.shuf.1e3.bed b.shuf.1e3.bed > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=2000m:ncpus=1 -m bea -M [email protected] -N bx-e3-full
echo "cd $PBTHOME; runit interval_join.py a.shuf.1e4.bed b.shuf.1e4.bed > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=4000m:ncpus=1 -m bea -M [email protected] -N bx-e4-full
echo "cd $PBTHOME; runit interval_join.py a.shuf.1e5.bed b.shuf.1e5.bed > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=8000m:ncpus=1 -m bea -M [email protected] -N bx-e5-full
echo "cd $PBTHOME; runit interval_join.py a.shuf.1e6.bed b.shuf.1e6.bed > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=16000m:ncpus=1 -m bea -M [email protected] -N bx-e6-full
echo "cd $PBTHOME; runit interval_join.py a.shuf.1e7.bed b.shuf.1e7.bed > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=64000m:ncpus=1 -m bea -M [email protected] -N bx-e7-full
echo "cd $PBTHOME; runit interval_join.py a.shuf.1e8.bed b.shuf.1e8.bed > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=90000m:ncpus=1 -m bea -M [email protected] -N bx-e8-full


####################################################
# bx-python uniq
####################################################
echo "cd $PBTHOME; runit bed_intersect.py a.shuf.1e3.bed b.shuf.1e3.bed > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=2000m:ncpus=1 -m bea -M [email protected] -N bx-e3-uniq
echo "cd $PBTHOME; runit bed_intersect.py a.shuf.1e4.bed b.shuf.1e4.bed > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=4000m:ncpus=1 -m bea -M [email protected] -N bx-e4-uniq
echo "cd $PBTHOME; runit bed_intersect.py a.shuf.1e5.bed b.shuf.1e5.bed > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=8000m:ncpus=1 -m bea -M [email protected] -N bx-e5-uniq
echo "cd $PBTHOME; runit bed_intersect.py a.shuf.1e6.bed b.shuf.1e6.bed > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=16000m:ncpus=1 -m bea -M [email protected] -N bx-e6-uniq
echo "cd $PBTHOME; runit bed_intersect.py a.shuf.1e7.bed b.shuf.1e7.bed > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=64000m:ncpus=1 -m bea -M [email protected] -N bx-e7-uniq
echo "cd $PBTHOME; runit bed_intersect.py a.shuf.1e8.bed b.shuf.1e8.bed > /dev/null" | \
   qsub -q cphg -W group_list=CPHG -V -l select=1:mem=90000m:ncpus=1 -m bea -M [email protected] -N bx-e8-uniq