Jupyter notebook as Slurm job

slurm_jupyter.md

Create slurm_jupyter.sbatch

#!/bin/bash
#SBATCH --nodes 1
#SBATCH --ntasks-per-node 20
#SBATCH --exclusive
#SBATCH --time 4:00:00
#SBATCH --mem-per-cpu 4000
#SBATCH --job-name jupyter
#SBATCH --output jupyter-log-%J.txt

## get tunneling info
XDG_RUNTIME_DIR=""
ipnport=$(shuf -i8000-9999 -n1)
ipnip=$(hostname -i)

## print tunneling instructions to jupyter-log-{jobid}.txt
echo -e "
    Create the following entry in your .ssh/config:
    -----------------------------------------------------------------
    Host cluster
        HostName 52.151.41.95
        User NORTHAMERICA.<your-microsoft-id>
    -----------------------------------------------------------------

    Copy/Paste this in your local terminal to ssh tunnel with remote
    -----------------------------------------------------------------
    ssh -N -L 9999:$ipnip:$ipnport cluster
    -----------------------------------------------------------------

    Then open a browser on your local machine to the following address
    ------------------------------------------------------------------
    localhost:9999  (prefix w/ https:// if using password)
    ------------------------------------------------------------------
    "

## start an ipcluster instance and launch jupyter server
jupyter-notebook --no-browser --port=$ipnport --ip=$ipnip --NotebookApp.token=''

Run sbatch slurm_jupyter.sbatch and wait for a file jupyter-log-<some number>.txt to show up.

cat jupyter-log-<some number>.txt and follow the instructions!