Plot GPU compute and memory utilization curve

plot-cuda-util.sh

#!/usr/bin/env bash

fname=$1

# if the log file does not exist yet, record it (stop by pressing ctrl+c)
if ! test -f $fname; then
    nvidia-smi -f $fname -l 1 --format=csv --query-gpu=timestamp,memory.total,memory.used,utilization.gpu,utilization.memory &

    bg=$!

    read -p "press return to stop recording $fname and start plotting to ${fname:0:${#fname}-4}.png "	

    kill -INT $bg
fi

# run actual plot script
python - <<"EOF" $fname
import matplotlib.pyplot as plt
import pandas as pd
import sys

fname=sys.argv[1]

def intdropunit(s):
    # get rid of units (" MiB", " %")
    return int(s.split()[0])

# parse the CSV
with open(fname) as file:
    log = pd.read_csv(file, 
        parse_dates=['timestamp'], 
        converters={1: intdropunit, 2: intdropunit, 3: intdropunit, 4: intdropunit}
    )

# filter non-zero (i.e. non-min) rows
log = log[~(log.iloc[:,1:] == log.min(axis="index")[1:]).all(axis="columns")]

# subplot for MiB and subplot for %
fig, axs = plt.subplots(2, sharex=True, dpi=600)
log.plot('timestamp', [1, 2], ax=axs[0])
log.plot('timestamp', [3, 4], ax=axs[1])
fig.savefig(fname[:-4] + '.png')

EOF

ls ${fname:0:${#fname}-4}.png

example output:

(very low average utilisation: too small batch size and GPU waiting on CPU)