use nvidia-smi to find available GPU(s)

find-gpu

#! /bin/bash -f
set -e

k=1
s=1

function usage() {
    echo "find-gpu [K [SECS]]"
    echo " Prints the indices of the K (default: ${k}) least-used"
    echo " GPUs, as a comma-separated list. Symmetries broken"
    echo " randomly. Sleeps randomly between 0 and SECS (default: ${s})"
    echo " seconds before running, to further break symmetry and"
    echo " avoid races."
}

if [ $# = 1 -a x$1 = x-h ]; then
    usage
    exit 0
fi

case $# in
    0) ;;
    1) k=$1 ;;
    2) k=$1; s=$2 ;;
    *) usage; exit 1 ;;
esac

sleep $(echo "scale=3; ${s}*${RANDOM}/32767" | bc)
nvidia-smi --format=csv --query-gpu=index,utilization.memory,utilization.gpu \
 | tail +2 \
 | tr -d '%' \
 | awk 'BEGIN {FS=","; srand()} {print $1, $2+$3+0.01*rand()}' \
 | sort -n --key=2 \
 | cut --fields=1 --delimiter=' ' \
 | head -${k} \
 | fmt \
 | tr ' ' ','