run bash commands in parallel

wait_run_in_parallel.sh

# This is a simple way to run multiple things in parallel in a bash
# script without making things too complicated
# it's modeled after (and uses) the bash `wait` command
# TIP: to do more complicated steps in parallel put them in a function

function wait_run_in_parallel()
{
    local number_to_run_concurrently=$1
    if [ `jobs -np | wc -l` -gt $number_to_run_concurrently ]; then
        wait `jobs -np | head -1` # wait for the oldest one to finish
    fi
}

function do_work()
{
    local msg=$1
    local sleep_time=$(($RANDOM % 10))
    echo "starting $msg"
    sleep $sleep_time && echo $msg
}

for x in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15; do

     # end a statement with a normal '&' to background it
     do_work "done $x" &

     # now wait if there are more than N sub processes executing
     wait_run_in_parallel 5
done
wait # for all bg commands to be done

GNU Parallel seems to solve the same problem.

parallel gzip ::: *.log

Watch the intro video to learn more: http://www.youtube.com/watch?v=OpaiGYxkSuQ