Avoid QoS error when submitting large number of slurm tasks. The script dispatch new task when previous one is completed.

slurm-feeder.sh

#!/usr/bin/env bash
set -euo pipefail

if [[ $# -lt 1 ]]; then
    echo "Usage: $0 COMMAND_FILE" >&2
    exit 1
fi

CMD_FILE=$1
MAX_JOBS=${MAX_JOBS:-20}
SLEEP_SECONDS=${SLEEP_SECONDS:-60}

if [[ ! -f "$CMD_FILE" ]]; then
    echo "Command file not found: $CMD_FILE" >&2
    exit 1
fi

current_job_count() {
    squeue -h -u "$USER" | wc -l
}

while IFS= read -r cmd || [[ -n "$cmd" ]]; do
    trimmed=${cmd//[[:space:]]/}
    [[ -z "$trimmed" ]] && continue
    stripped=${cmd#${cmd%%[![:space:]]*}}
    [[ ${stripped:0:1} == "#" ]] && continue
    while (( $(current_job_count) >= MAX_JOBS )); do
        sleep "$SLEEP_SECONDS"
    done
    echo "Submitting: $cmd"
    if ! eval "$cmd"; then
        echo "Submission failed: $cmd" >&2
        exit 1
    fi
    sleep 1
done <"$CMD_FILE"

echo "All commands from $CMD_FILE submitted."