nvidia-smi

smi

#!/usr/bin/bash
set -e
echo "PID     GPU        MEM  USER          TTY STAT  START    TIME  COMMAND"; 
join -1 3 -2 1 <( nvidia-smi | grep -A100 PID | sed -n '3,$p' | grep -v -- ------- | awk '{print $2,$6,$3}'  | sort -k3,3 ) <( LC_ALL=C ps aux | awk '{print $2,$1,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20}' | sort -k1,1 ) | sort -k 2,2 | while read -r LINE ; do
    printf "%-7s%2s%13s  %-10s%7s%5s%7s%8s  %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s\n" $LINE
        done | awk '{print $0; gsub("MiB","",$3); u[$2]+=$3}END{print "\nGPU      MEM    USAGE";for(i=0;i<8;i++){printf "%2s %7s MiB  %.1f%%\n",i,u[i],u[i]/244.78}}' | cut -b1-`tput cols`
echo