bash parallelization template

parallel.sh

#!/bin/bash
# Parallelize the execution of a command
######
CHUNK_COUNT=0

parallel_configure () {
  if [ $# -eq 0 ]; then
    echo "usage: <tmp dir> <tmp prefix> <num chunks>"
  fi

  TMP_DIR=${1:-"/tmp/work"}
  TMP_PREFIX=${2:-"$TMP_DIR/tmp_file"}
  NUM_CHUNKS=${3:-4}

  [ ! -d "$TMP_DIR" ] && mkdir -p "$TMP_DIR";
}

# call w/o args to use defaults
parallel_configure

__advance_count () {
  ((CHUNK_COUNT+=1))
}

__get_tmp_file () {
  echo "$TMP_PREFIX$CHUNK_COUNT.tmp"
}

__get_err_file () {
  echo "$TMP_PREFIX$CHUNK_COUNT.err"
}

__to_parallel () {
  $@ & 1> "$(__get_tmp_file)" 2>"$(__get_err_file)"
  return $?
}

parallel_run () {

  local pids=()
  for i in {0..$NUM_CHUNKS}; do
    __to_parallel "$@"
    pids+=("$!")
  
    __advance_count
  done

  # wait for the processes
  for pid in $pids; do wait $pid; done
  
  # concatenate tmp files
  cat $TMP_PREFIX*.tmp > 1
  cat $TMP_PREFIX*.err > 2
}