Running jobs in parallel, then merge in bash with the command "wait"

bsub.sh

#!/bin/sh
#
# Submit a single test on LSF
#  Wrapper around bsub command, then process log result.
#
# This script is used by run-tests-async.sh
#
# Usage:
#
#  ./bsub.sh $command

index=$1
cmd=$2

# Set project code
PROJECT_CODE=TRAIN

# Set standard output log and error log
log_dir=$(pwd)/logs
log=$log_dir/bsub_${index}.log
err=$log_dir/bsub_${index}.err

# Capture job summary
run_log=$log_dir/summary.log

# Clear log
echo " " > ${log}
echo " " > ${err}

# Job requierement
rselect="select[rhe6 && os64 && linux && mem > 2048 && swp > 10240]"

# BSUB command (blocking mode: -K)
bsub -P $PROJECT_CODE -W 720 -R "${rselect}" -J bsub_${index} -q low -K -o ${log} -e ${err} ${cmd} > ${log}_bsub 2>&1

# Post processing
#
# Check if log file is present
if [ ! -f ${log} ]; then
  echo "ERROR: ${log} log does not exist !" | tee -a $run_log
  exit 2
fi

# Check content
if grep 'OK' ${log} > /dev/null 2>&1;
  then
      echo "OK: ${log} : (${index})" | tee -a $run_log
      exit 0
  else
      echo "FAIL: ${log} (${index})" | tee -a $run_log
      exit 1
  fi
fi

cmd.txt

ls
sleep 10s
ls -1

main.sh

#!/bin/sh
#
# Submit a single test on LSF
#  Wrapper around bsub command, then process log result.
#
# This script is used by run-tests-async.sh
#
# Usage:
#
#  ./bsub.sh $command

index=$1
cmd=$2

# Set project code
PROJECT_CODE=TRAIN

# Set standard output log and error log
log_dir=$(pwd)/logs
log=$log_dir/bsub_${index}.log
err=$log_dir/bsub_${index}.err

# Capture job summary
run_log=$log_dir/summary.log

# Clear log
echo " " > ${log}
echo " " > ${err}

# Job requierement
rselect="select[rhe6 && os64 && linux && mem > 2048 && swp > 10240]"

# BSUB command (blocking mode: -K)
bsub -P $PROJECT_CODE -W 720 -R "${rselect}" -J bsub_${index} -q low -K -o ${log} -e ${err} ${cmd} > ${log}_bsub 2>&1

# Post processing
#
# Check if log file is present
if [ ! -f ${log} ]; then
  echo "ERROR: ${log} log does not exist !" | tee -a $run_log
  exit 2
fi

# Check content
if grep 'OK' ${log} > /dev/null 2>&1;
  then
      echo "OK: ${log} : (${index})" | tee -a $run_log
      exit 0
  else
      echo "FAIL: ${log} (${index})" | tee -a $run_log
      exit 1
  fi
fi

parallel_bsub.sh

#!/bin/sh
#
# Run jobs in parallel, then wait for completion of all of them.
#
# Features:
#
#  - Run tests in parallel
#  - Post processing step
#  - Easy to manage large number of parallel jobs
#
# Usage:
#
# ./main.sh cmd.txt
#

[[ $# -eq 1 ]] ||
{
    echo "File with the list of commands to execute as LSF JOB."
    exit 1
}

file=$1 # txt file with the list of commands to run
index=0 # line number

while read line
do
  # Increment index
  index=$((index+1))

  # Run bsub as UNIX job
  ./bsub.sh ${index} "${line}" &

done < $file

echo ""
echo "Waiting jobs..."
wait
echo "All jobs completed..."
echo ""

exit 0