Nucleus nextflow ignite sbatch script

ignite_batch.sh

#!/bin/bash
#SBATCH --job-name=ignite-nf-wordcount
#SBATCH --output=nf_ignite_%j.log

# Create the ignite allocation across 4 32GB nodes
#SBATCH --partition=32GB
#SBATCH -N 4

# This creates a random cluster seed, so the ignite cluster can communicate
# with itself and not clash with other running ignite clusters on Nucleus.
export NXF_CLUSTER_SEED=$(shuf -i 0-16777216 -n 1)

module add nextflow

# We need to provide user writable locations for nextflow to initialize its
# Ignite dependencies etc. for this job.
#
# Nextflow is designed to bring down latest non-core dependencies on the fly,
# so it's better to do this than adding a fixed set into the BioHPC modules.
export NXF_ASSETS="$(pwd)/nf_ignite_${NXF_CLUSTER_SEED}/assets"
export NXF_LAUNCHER="$(pwd)/nf_ignite_${NXF_CLUSTER_SEED}/launcher"
export CAPSULE_CACHE_DIR="$(pwd)/nf_ignite_${NXF_CLUSTER_SEED}/capsule"

# Multicast cluster discovery may be unreliable with the multiple network
# interfaces and subnets in the Nucleus cluster, so we need to use a directory
# to co-ordinate the cluster.
export CLUSTER_STATE_DIR="$(pwd)/nf_ignite_${NXF_CLUSTER_SEED}/state"
mkdir -p "${CLUSTER_STATE_DIR}"

# Nextflow ignite implementation runs process at the master, as well as on the
# Ignite workers, so we need to run N-1 workers, and 1 master - rather than N
# workers and 1 master.
let NUM_WORKERS="$SLURM_JOB_NUM_NODES -1"

echo "Will run ${NUM_WORKERS} + 1 master across allocation of ${SLURM_JOB_NUM_NODES} nodes."

# Start up N-1 workers
srun --exclusive -n ${NUM_WORKERS} --cpus-per-task=${SLURM_CPUS_ON_NODE} --distribution=cyclic nextflow node -cluster.join path:${CLUSTER_STATE_DIR} &

# Start 1 master
srun --exclusive -n 1 --cpus-per-task=${SLURM_CPUS_ON_NODE} --distribution=cyclic nextflow run main.nf -process.executor ignite -cluster.join path:${CLUSTER_STATE_DIR} -with-trace -with-timeline