For each GPU, list used memory by process (and find associated docker container)

gpuinfo.sh

#!/bin/bash

for i in $(nvidia-smi --query-gpu=index --format=csv,noheader); do
    echo GPU "#$i": $(nvidia-smi --query-gpu=memory.used --format=csv,noheader -i $i) used

    while IFS="," read -r pid name used; do
        pid_container_name=
        for container in $(docker ps -q); do
            if [[ $(docker top $container | awk '{print $2}'| grep $pid) ]]; then
                pid_container_name=$(docker inspect $container --format '{{.Name}}')
            fi
        done
        cmd=$(ps -p $pid -o cmd=)
        if [ -z "$cmd" ]; then
            cmd=$name
        fi
        if [[ $pid_container_name ]]; then
            pid_container_name="in $pid_container_name"
        fi
        echo "       " $used used by PID $pid $pid_container_name running $cmd    
    done < <(nvidia-smi --query-compute-apps=pid,process_name,used_gpu_memory -i $i --format=csv,noheader)
    echo
done