iterative resource allocation

doubling-qsub.sh

#!/bin/bash

dir=$(dirname $(readlink -ef $0))
cg=/sys/fs/cgroup
mem=250000                   # 250MB
maxmem=2000000                  # 2GB: *8
time=225                     # 3.75 min = 30/4
maxtime=1800                    # 30 min
options=
debug=false
cgname=

while getopts ":g:m:M:t:T:o:d" opt
do
    case ${opt} in
        g) cgname=${OPTARG} ;; # group name
        m) mem=${OPTARG:=$mem} ;;
        M) maxmem=${OPTARG:=$maxmem} ;;
        t) time=${OPTARG:=$time} ;;
        T) maxtime=${OPTARG:=$maxtime} ;;
        o) options=${OPTARG} ;;
        d) debug=true ;;
        \?) OPT_ERROR=1; break;;
        * ) echo "unsupported option $opt" ;;
    esac
done

optionfile=$(mktemp)
echo $options > $optionfile

if [[ $cgname == "" ]]
then
    echo "ERROR: cgroup name is not specified" >&2
    echo "ERROR: defaulting to '$(whoami)'" >&2
    cgname=$(whoami)
fi 
echo "Ensuring cgroup $cgname exists."
cgcreate -t $(whoami):$(whoami) -a $(whoami):$(whoami) -g cpuacct,memory:$cgname  # might complain, but usually it is successful

shift $(($OPTIND - 1))

command=$(readlink -ef $1)
. utilities.sh

next

iterator.sh

#!/bin/bash

. $dir/utilities.sh

ccgname=$cgname/$$            # child cgname
cg=/sys/fs/cgroup
cgcpu=$cg/cpuacct/$ccgname
cgmem=$cg/memory/$ccgname
mkdir -p $cgcpu
mkdir -p $cgmem

pushd $cgmem &> /dev/null
echo 0 > memory.swappiness
popd &> /dev/null

pid=

sleep=10
mykill (){
    echodo kill -s SIGXCPU $pid
    sleep $sleep
    ps $pid &> /dev/null && {
        echodo kill -s SIGTERM $pid
        sleep $sleep
        ps $pid &> /dev/null && {
            echodo kill -s SIGKILL $pid
        }
    }
}

twice-time (){
    echo "Current iteration failed! Doubling the time..." >&2
    ntime=$(( 2 * $time ))
    if [[ $ntime -gt $maxtime ]]
    then
        echo "Failed on time=$time, maxtime=$maxtime, no more iteration!" >&2
        exit 1
    fi
    time=$ntime
    next
}

twice-mem (){
    echo "Current iteration failed! Doubling the memory..." >&2
    nmem=$(( 2 * $mem ))
    if [[ $nmem -gt $maxmem ]]
    then
        echo "Failed on mem=$mem, maxmem=$maxmem, no more iteration!" >&2
        exit 1
    fi
    mem=$nmem
    next
}

finalize (){
    echo
    echo real $(($(< $cgcpu/cpuacct.usage) / 1000000))
    echo maxmem $(( $(< $cgmem/memory.max_usage_in_bytes) / 1024 ))
    rmdir $cgcpu
    rmdir $cgmem
}

trap finalize EXIT

echo Current resource limit: time: $time memory: $mem
cgexec -g cpuacct,memory:$ccgname $command && rm -f $optionfile &
pid=$!

while ps $pid &> /dev/null
do
    sleep 1
    cpuusage=$(($(< $cgcpu/cpuacct.usage) / 1000000))
    if [[ $cpuusage -gt ${time}000 ]]
    then
        echo "cpuacct.usage exceeding. $cpuusage msec."
        mykill
        twice-time
        break
    fi
    memusage=$(( $(< $cgmem/memory.max_usage_in_bytes) / 1024 ))
    if [[ $memusage -gt $mem ]]
    then
        echo "memory.max_usage_in_bytes exceeding. $memusage kB."
        mykill
        twice-mem
        break
    fi
done

setup-cgroups.sh

#!/bin/bash

sudo cgcreate -a $(whoami):$(whoami) -t $(whoami):$(whoami) -g cpuacct,memory:$(whoami)

tarai.lisp

#!/usr/local/bin/sbcl --script

(print "ITERATOR TEST")
(force-output)

(defun tarai (x y z)
  (if (<= x y)
      y
      (tarai (tarai (1- x) y z)
             (tarai (1- y) z x)
             (tarai (1- z) x y))))

(time (print (tarai 15 8 1)))

test.sh

#!/bin/bash


./doubling-qsub.sh -t 1 -T 100 -m 1000000 -M 1000000 tarai.lisp
# 1MB

utilities.sh

#!/bin/bash

echovar (){
    echo -n $1 " "
    eval "echo \$$1"
}

echodo (){
    echo $*
    $@
}

kv(){
    eval "echo -n \"$1='\$$1'\""
}

kvs(){
    first=true
    for x in $@
    do
        if $first
        then
            first=false
            echo -n "-v "
        else
            echo -n ","
        fi
        kv $x
    done
}

dkv(){                          # debug
    eval "echo -n \"$1=\$$1 \""
}

dkvs(){                         # debug
    for x in $@ ; do dkv $x ; done
}

next(){
    # echo "Iterating... debug:$debug"
    if $debug
    then
        eval "$(dkvs mem maxmem time maxtime cgname debug dir) command='$command' $dir/iterator.sh 2>&1 | tee ${command%%.*}-mem$mem-time$time"
    else
        cd $dir
        echodo qsub -l mem=$mem,pmem=$mem,walltime=$(( 20 + $time )),nodes=1:ppn=$((($mem + 1999999) / 2000000)) \
            $(kvs mem maxmem time maxtime cgname debug dir command optionfile) \
            $(eval "echo $(< $optionfile)") iterator.sh
    fi
}