How to use multiple queues for passing data to/from a multiprocessing pool.

queue_pool_demo.py

#!/usr/bin/env python

"""
How to use multiple queues for passing data to/from a multiprocessing pool.
"""

from multiprocessing import Pool, Queue
import shortuuid

def f(q_in, q_out):
    """ 
    Worker process body.
    """

    # Create unique ID to clearly distinguish between workers:
    u = shortuuid.uuid()
    while True:
        m = q_in.get()
        print '%s: %s' % (u, m)
        if m == 'q':
            break
        else:
            q_out.put(-m)

if __name__ == '__main__':
    N = 4 
    q_in = Queue()
    q_out = Queue()
    p = Pool(N, f, (q_in, q_out))

    # Send input:
    M = 10
    for i in xrange(M):
        q_in.put(i)

    # Get output:
    for i in xrange(M):
        m = q_out.get()
        print m

    # Tell workers to quit:
    for i in xrange(N):
        q_in.put('q')