scratch tools

backup.sh

while true
do
    sleep 300
    tar cf - --directory /scratch/$USER/local pytorch | lz4 - -f /scratch/$USER/local-pytorch.lz4 -q
    mv /scratch/$USER/local-pytorch.lz4 /data/home/$USER/scratch_tools/
done

init.sh

#!/bin/bash
CONDA_ENV=scratch39
source /etc/profile
source /data/home/$USER/.bashrc

DO_COPY=0
DO_AUTOMATIC_BACKUP=1

while [[ $# -gt 0 ]]; do
    case $1 in
        -h|--help)
          echo "--copy: force copy the backed-up lz4 file"
          shift
          ;;
        -c|--copy)
          DO_COPY=1
          echo "Copying data to compile node"
          shift
          ;;
        -s|--skip-automatic-backup)
          DO_AUTOMATIC_BACKUP=0
          echo "Will skip automatic backups"
          shift
          ;;
    esac
done

eval "$(conda shell.bash hook)"

SCRIPT_FNAME=$(mktemp)
cat > $SCRIPT_FNAME <<EOT
#!/bin/bash
DO_COPY="$DO_COPY"
DO_AUTOMATIC_BACKUP="$DO_AUTOMATIC_BACKUP"
source /etc/profile
source /data/home/\$USER/.bashrc
eval "\$(conda shell.bash hook)"
echo \$USER
echo activating conda env
conda activate "${CONDA_ENV}"
if [[ "\$DO_COPY" == "1" ]]; then
    mkdir -p /scratch/\$USER/local
    echo copying data...
    cp /data/home/\$USER/scratch_tools/local-pytorch.lz4 /scratch/\$USER/
    echo unzipping
    lz4 -dc < /scratch/\$USER/local-pytorch.lz4 | tar xf - -C /scratch/\$USER/local/
fi
function start_periodic_backup() {
    if [[ "\$DO_AUTOMATIC_BACKUP" == "1" ]]; then
        bash /data/home/\$USER/scratch_tools/backup.sh &
        SCRATCH_BACKUP_PID=\$!
    fi
}
echo set backup for every 5 minutes
start_periodic_backup

function stop_periodic_backup() {
    if [[ "\$DO_AUTOMATIC_BACKUP" == "1" ]]; then
        kill \$SCRATCH_BACKUP_PID
    fi
}

function run_backup() {
    stop_periodic_backup
    echo tarring and compressing
    time tar cf - --directory /scratch/\$USER/local pytorch | lz4 - -f /scratch/\$USER/local-pytorch-inprogress.lz4
    echo copying back to NFS
    time mv /scratch/\$USER/local-pytorch-inprogress.lz4 /data/home/\$USER/scratch_tools/
    echo replacing old backup with new backup
    time mv /data/home/\$USER/scratch_tools/local-pytorch-inprogress.lz4 /data/home/\$USER/scratch_tools/local-pytorch.lz4
    start_periodic_backup
}

function run_final_backup() {
    echo "stopping the periodic backup so it doesn't interfere with the final backup..."
    stop_periodic_backup
    echo running the final backup...
    if [[ "\$DO_AUTOMATIC_BACKUP" == "1" ]]; then
        run_backup
    fi
}

trap run_final_backup exit
EOT

cat $SCRIPT_FNAME
bash --init-file $SCRIPT_FNAME