A simple script to run a command as soon as a GPU is idle.

queue_gpu.py

import subprocess
import time
import re
import sys

while True:
    out = subprocess.check_output('nvidia-smi')
    usage = int(re.search(b'\d*MiB /', out).group(0).split(b'M')[0])
    if usage < 1000:
        subprocess.Popen(sys.argv[1:])
        break
    time.sleep(1)