GPU Monitoring with Cribl: nvidia-smi Command & Cribl Search Examples

nvidia-smi.sh

nvidia-smi --query-gpu=timestamp,index,name,uuid,driver_version,pci.bus_id,persistence_mode,power.draw,power.limit,temperature.gpu,fan.speed,utilization.gpu,utilization.memory,utilization.encoder,utilization.decoder,memory.total,memory.used,memory.free,clocks.sm,clocks.mem --format=csv,noheader,nounits | while IFS=, read -r ts idx name uuid drv pci persist pdraw plimit temp fan ugpu umem uenc udec mtot mused mfree csm cmem; do iso_ts=$(date -d "$ts" --utc +%Y-%m-%dT%H:%M:%SZ); echo "{\"timestamp\":\"$iso_ts\",\"index\":$idx,\"name\":\"$name\",\"uuid\":\"$uuid\",\"driver_version\":\"$drv\",\"pci_bus_id\":\"$pci\",\"persistence_mode\":\"$persist\",\"power_draw\":$pdraw,\"power_limit\":$plimit,\"temperature_gpu\":$temp,\"fan_speed\":$fan,\"utilization_gpu\":$ugpu,\"utilization_memory\":$umem,\"utilization_encoder\":$uenc,\"utilization_decoder\":$udec,\"memory_total\":$mtot,\"memory_used\":$mused,\"memory_free\":$mfree,\"clocks_sm\":$csm,\"clocks_mem\":$cmem}"; done

For dashboards, see: https://gist.github.com/hyfather/dfe7c36c3ba9cb893712782bf43a5e97