nvprof at scale.txt

#!/bin/bash                                                                                                      
world_rank=$PMIX_RANK
let local_size=$RANKS_PER_SOCKET
export CUDA_CACHE_PATH=/dev/shm/$USER/nvcache_$PMIX_RANK
executable=$1
shift
if [ $world_rank = $PROFILE_RANK ]; then
    nvprof -f -o $PROFILE_PATH $executable "$@"
else
    $executable "$@"
fi
rm -rf /dev/shm/$USER/nvcache_$PMIX_RANK

#####################################################
#Then in the script you use to submit to jsrun, you would have these lines:

export PROFILE_RANK=1  #rank where helper script will run nvprof
export PROFILE_PATH=/gpfs/alpinetds/scratch/dappelh/csc275/nvp.prof
echo "nvprof output at $PROFILE_PATH"

jsrun <all the normal jsrun stuff> helper.sh myprogram.exe