msub-batch.sh

#!/bin/env bash

# Usage: msub-batch.sh -l walltime=73:00:00 [-w 43200] -- -l nodes=1:gpus=2
#   Automatically submits a series of jobs that are dependent on each other.
#   By default it submits jobs of 24 hours, but a different interval can be
#   given using the -w flag (in seconds).  Each job can look at the MOAB_JOBID
#   and MOAB_DEPEND variables to see what its own ID are and whether it has
#   dependencies

MAX_RUNTIME=$((24 * 60 * 60))

# Calculate how many jobs we need
while getopts "w:l:" opt; do
  case $opt in
    l)
      WALLTIME=${OPTARG#walltime=}
      ;;
    w)
      MAX_RUNTIME=$OPTARG
      ;;
  esac
done

shift $((OPTIND - 1))

if [[ $WALLTIME == *":"* ]]; then
  HOURS=${WALLTIME%:*:*}
  TMP=${WALLTIME#*:}
  MINUTES=${TMP%:*}
  SECONDS=${TMP#*:}
  TOTAL=$((HOURS * 60 * 60 + $MINUTES * 60 + $SECONDS))
else
  TOTAL=$WALLTIME
fi

NUM_JOBS=$(($TOTAL / $MAX_RUNTIME))
if [[ $(($TOTAL % $MAX_RUNTIME)) -gt 0 ]]; then
  NUM_JOBS=$((NUM_JOBS + 1))
fi

# Submit the first job (no dependency)
if [[ $TOTAL -gt MAX_RUNTIME ]]; then
  echo msub -E -l walltime=$MAX_RUNTIME "$@"
  JOBID=$(msub -E -l walltime=$MAX_RUNTIME "$@")
  JOBID=${JOBID##*[[:space:]]}

  # Submit any following 24-hour blocks
  for i in $(seq 1 $(($NUM_JOBS - 2))); do
    echo msub -E -l walltime=$MAX_RUNTIME -l depend=$JOBID "$@"
    JOBID=$(msub -E -l walltime=$MAX_RUNTIME -l depend=$JOBID "$@")
    JOBID=${JOBID##*[[:space:]]}
  done
  
  # Submit the last job
  echo msub -E -l walltime=$(($TOTAL - $MAX_RUNTIME * ($NUM_JOBS - 1))) -l depend=$JOBID "$@"
  msub -E -l walltime=$(($TOTAL - $MAX_RUNTIME * ($NUM_JOBS - 1))) -l depend=$JOBID "$@"
else
  echo msub -E -l walltime=$((TOTAL)) "$@"
  msub -E -l walltime=$((TOTAL)) "$@"
fi