Skip to content

Instantly share code, notes, and snippets.

@lrvick
Forked from ask/get_results.py
Created June 22, 2011 18:03
Show Gist options
  • Save lrvick/1040703 to your computer and use it in GitHub Desktop.
Save lrvick/1040703 to your computer and use it in GitHub Desktop.
Get async celery results from nested subtasks as they complete
from tasks import task1
def get_results(queries):
query_procs = task1.delay(queries).get().join()
results = []
for query_proc in query_procs:
# while the following iterate() is happening, the other query_procs are ignored.
# ideas on iterating over all of them at once?
for result in query_proc.iterate():
yield result
for result in get_results(['a','b','c']):
print result
from celery.task import task,TaskSet
import time
import random
@task
def task1(queries):
return TaskSet(task2.subtask((query, )) for query in queries).apply_async()
@task
def task2(query):
return TaskSet(task3.subtask(query) for _ in xrange(20)).apply_async()
@task
def task3(query):
seconds = random.randint(0,10)
time.sleep(seconds)
return query,seconds
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment