Measure time performance of slurp.{pl,py} scripts.

.gitignore

/1/
/2/

generate-input.sh

#!/bin/bash
mkdir 1
cd 1
for i in {1..50000}; do dd if=/dev/urandom of=$i.xml bs=1K count=10; done
cd ..
cp -r 1 2

measure-ramfs.log

+ TIME=/usr/bin/time
+ PERLCMD='perl slurp.pl'
+ PYTHONCMD='python slurp.py'
+ d1=/tmp/1
+ d2=/tmp/2
+ sudo -v
+ clearcache
+ sync
+ sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches'
+ /usr/bin/time perl slurp.pl /tmp/1
0.43user 0.37system 0:00.86elapsed 93%CPU (0avgtext+0avgdata 8928maxresident)k
6808inputs+0outputs (17major+2464minor)pagefaults 0swaps
+ clearcache
+ sync
+ sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches'
+ /usr/bin/time python slurp.py /tmp/1
0.50user 0.48system 0:01.06elapsed 92%CPU (0avgtext+0avgdata 10684maxresident)k
11488inputs+0outputs (33major+53504minor)pagefaults 0swaps
+ /usr/bin/time perl slurp.pl /tmp/2
0.46user 0.38system 0:00.93elapsed 90%CPU (0avgtext+0avgdata 8936maxresident)k
12744inputs+0outputs (16major+2477minor)pagefaults 0swaps
+ /usr/bin/time python slurp.py /tmp/2
0.48user 0.51system 0:01.00elapsed 99%CPU (0avgtext+0avgdata 10684maxresident)k
0inputs+0outputs (0major+53537minor)pagefaults 0swaps
+ /usr/bin/time perl slurp.pl /tmp/2
0.46user 0.31system 0:00.78elapsed 99%CPU (0avgtext+0avgdata 8928maxresident)k
0inputs+0outputs (0major+2493minor)pagefaults 0swaps
+ /usr/bin/time python slurp.py /tmp/2
0.48user 0.48system 0:00.96elapsed 99%CPU (0avgtext+0avgdata 10684maxresident)k
0inputs+0outputs (0major+53537minor)pagefaults 0swaps
+ /usr/bin/time perl slurp.pl /tmp/2
0.44user 0.31system 0:00.76elapsed 99%CPU (0avgtext+0avgdata 8928maxresident)k
0inputs+0outputs (0major+2492minor)pagefaults 0swaps
+ /usr/bin/time python slurp.py /tmp/2
0.44user 0.52system 0:00.97elapsed 99%CPU (0avgtext+0avgdata 10680maxresident)k
0inputs+0outputs (0major+53537minor)pagefaults 0swaps

measure.log

+ TIME=/usr/bin/time
+ PERLCMD='perl slurp.pl'
+ PYTHONCMD='python slurp.py'
+ d1=1
+ d2=2
+ sudo -v
+ clearcache
+ sync
+ sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches'
+ /usr/bin/time perl slurp.pl 1
1.46user 4.27system 0:19.46elapsed 29%CPU (0avgtext+0avgdata 8932maxresident)k
1232016inputs+0outputs (18major+2475minor)pagefaults 0swaps
+ clearcache
+ sync
+ sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches'
+ /usr/bin/time python slurp.py 1
1.03user 5.10system 0:18.21elapsed 33%CPU (0avgtext+0avgdata 10408maxresident)k
1239592inputs+0outputs (31major+53463minor)pagefaults 0swaps
+ /usr/bin/time perl slurp.pl 2
1.62user 4.51system 0:23.21elapsed 26%CPU (0avgtext+0avgdata 8928maxresident)k
1234760inputs+0outputs (16major+2476minor)pagefaults 0swaps
+ /usr/bin/time python slurp.py 2
0.54user 0.47system 0:01.02elapsed 99%CPU (0avgtext+0avgdata 10408maxresident)k
0inputs+0outputs (0major+53494minor)pagefaults 0swaps
+ /usr/bin/time perl slurp.pl 2
0.49user 0.32system 0:00.81elapsed 99%CPU (0avgtext+0avgdata 8928maxresident)k
0inputs+0outputs (0major+2492minor)pagefaults 0swaps
+ /usr/bin/time python slurp.py 2
0.51user 0.48system 0:01.00elapsed 99%CPU (0avgtext+0avgdata 10408maxresident)k
0inputs+0outputs (0major+53495minor)pagefaults 0swaps
+ /usr/bin/time perl slurp.pl 2
0.45user 0.34system 0:00.80elapsed 99%CPU (0avgtext+0avgdata 8932maxresident)k
0inputs+0outputs (0major+2492minor)pagefaults 0swaps
+ /usr/bin/time python slurp.py 2
0.47user 0.51system 0:00.99elapsed 99%CPU (0avgtext+0avgdata 10404maxresident)k
0inputs+0outputs (0major+53494minor)pagefaults 0swaps

measure.sh

#!/bin/bash
#
# Measure time performance of slurp.{pl,py} scripts.
#
# Note: call generate-input.sh once to generate input files
#
# Run, `sudo iotop -a -u me` to check that everything is read
# See https://stackoverflow.com/questions/26178038/python-slow-read-performance-issue
#
set -x -e
TIME=/usr/bin/time
PERLCMD="perl slurp.pl"
PYTHONCMD="python slurp.py"

d1="${1:-1}"
d2="${2:-2}"

# keep-alive sudo
# From https://gist.github.com/cowboy/3118588
# Might as well ask for password up-front, right?
sudo -v
# Keep-alive: update existing sudo time stamp if set, otherwise do nothing.
while true; do sudo -n true; sleep 60; kill -0 "$$" || exit; done 2>/dev/null &

function clearcache {
    sync; sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches'
}

# measure time performance
clearcache
$TIME $PERLCMD "$d1"
clearcache
$TIME $PYTHONCMD "$d1"

$TIME $PERLCMD "$d2"
$TIME $PYTHONCMD "$d2"
$TIME $PERLCMD "$d2"
$TIME $PYTHONCMD "$d2"
$TIME $PERLCMD "$d2"
$TIME $PYTHONCMD "$d2"

slurp-scandir.py

import sys
import scandir # $ pip install scandir
for entry in scandir.scandir(sys.argv[1]):
    if entry.name.endswith('.xml'):
        with open(entry.path) as x:
            f = x.read()

slurp.pl

foreach my $file (<$ARGV[0]/*.xml>){
    my $fh;
    open($fh, "< $file");
    my $contents = do { local $/; <$fh> };
    close($fh);
}

slurp.py

import glob
import os
import sys

L = glob.glob(os.path.join(sys.argv[1], '*.xml'))
L.sort(key=lambda s: int(os.path.splitext(os.path.basename(s))[0]))
for file in L:
    with open(file) as x:
        f = x.read()