GPU_load_statistic

gpu_analysis.py

"""
Usage:
    python gpu_analysis.py <gpu_load.txt> <gpu_num>
"""
import sys

gpu_num = int(sys.argv[2])
total = 0
utilized = 0
for line in open(sys.argv[1]).readlines():
    line = line.rstrip()
    data = line.split()[2:]
    assert len(data) == gpu_num
    data = [1 if int(x)>0 else 0 for x in data]
    total += len(data)
    utilized += sum(data)
print('utilization: %.3f' % (utilized / total))

gpu_load.sh

# Usage:
#     nohup bash gpu_load.sh &> gpu_load.txt &
while true;
do
t=`date --rfc-3339=second`;
u=`nvidia-smi | grep % | awk -F'|' '{print $4}' | awk -F'%' '{print $1}'`;
u=`echo $u | tr -s "\n" " "`
echo "$t $u";
sleep 1;
done