IPetrik

linar-jether / dask_celery_scheduler.py

Created March 18, 2018 09:20

A dask distributed scheduler based on on Celery tasks - Allows reusing an existing celery cluster for ad-hoc computation

	from __future__ import absolute_import, division, print_function

	import multiprocessing
	import pickle
	from multiprocessing.pool import ThreadPool

	from celery import shared_task
	from dask.local import get_async # TODO: get better get
	from dask.context import _globals
	from dask.optimize import fuse, cull