Script to send GPU monitoring data obtained with nvidia-smi to Stackdriver.

gpu_monitoring.py

from subprocess import Popen, PIPE
import os
import time
import sys

def compute_stats():
    all_gpu = []
    all_mem = []
    for i in range(10):
        p = Popen(["nvidia-smi","--query-gpu=utilization.gpu,utilization.memory", "--format=csv,noheader,nounits"], stdout=PIPE)
        stdout, stderror = p.communicate()
        output = stdout.decode('UTF-8')
        # Split on line break
        lines = output.split(os.linesep)
        numDevices = len(lines)-1
        gpu = []
        mem = []
        for g in range(numDevices):
            line = lines[g]
            #print(line)
            vals = line.split(', ')
            #print(vals)
            gpu.append(float(vals[0]))
            mem.append(float(vals[1]))

        all_gpu.append(gpu)
        all_mem.append(mem)
        time.sleep(1)

    max_gpu = [max(x[i] for x in all_gpu) for i in range(numDevices)]
    avg_gpu = [sum(x[i] for x in all_gpu)/len(all_gpu) for i in range(numDevices)]
    max_mem = [max(x[i] for x in all_mem) for i in range(numDevices)]
    avg_mem = [sum(x[i] for x in all_mem)/len(all_mem) for i in range(numDevices)]
    return max_gpu, avg_gpu, max_mem, avg_mem

from google.cloud import monitoring_v3

client = monitoring_v3.MetricServiceClient()
project = 'yourGCPProjectID'
project_name = client.project_path(project)


def write_time_series(name, gpu_nr, value):
    series = monitoring_v3.types.TimeSeries()
    series.metric.type = 'custom.googleapis.com/' + name
    series.resource.type = 'gce_instance'
    series.resource.labels['instance_id'] = sys.argv[1] + "_gpu_" + str(gpu_nr)
    series.resource.labels['zone'] = 'us-central1-f'

    point = series.points.add()
    point.value.double_value = value
    now = time.time()
    point.interval.end_time.seconds = int(now)
    point.interval.end_time.nanos = int(
    (now - point.interval.end_time.seconds) * 10**9)
    client.create_time_series(project_name, [series])

if len(sys.argv) < 2:
    print("You need to pass the instance name as first argument")
    sys.exit(1)

try:
    max_gpu, avg_gpu, max_mem, avg_mem = compute_stats()
    for i in range(len(max_gpu)):
        write_time_series('max_gpu_utilization', i, max_gpu[i])
        write_time_series('max_gpu_memory', i, max_mem[i])
        write_time_series('avg_gpu_utilization', i, avg_gpu[i])
        write_time_series('avg_gpu_memory', i, avg_mem[i])
except Exception as e:
    print(e)