a simple example for getting dask to work on a cluster with SLURM job queue

test_dask_1.py

"""
To install 
conda create -n dask-tutorial python=3.7 anaconda
conda activate dask-tutorial
conda install dask
conda install -c conda-forge dask-jobqueue
"""

import dask
import time
from dask.distributed import Client, progress, get_worker
from dask_jobqueue import SLURMCluster

# create a SLURMCluster instance with extra lines to load the Anaconda module
cluster = SLURMCluster(processes=1, cores=1, memory='4GB', walltime='1:00:00',
                       job_script_prologue=['#!/bin/bash', 'module load anaconda'])


cluster.scale(10) # this may take a few seconds to launch

# define a Dask function
def slow_increment(x):
    time.sleep(1)
    return(x+1)

# start client
client = Client(cluster)
futures = client.map(slow_increment, range(500) )
results = client.gather(futures)
print(results)
futures = client.map(slow_increment, range(500, 1000, 1))
results = client.gather(futures)
print(results)

# wait for the task to complete and retrieve the result

# close the cluster and the client
cluster.close()
client.close()