A convenient pattern for spawning worker threads and a queue in Python

worker_queue.py

import logging
from contextlib import contextmanager
from Queue import Queue
from threading import Thread


class ThreadStop(Exception):
    pass


def stop_queue():
    """Procedure to create a work stop sentinel."""
    raise ThreadStop()


def create_worker(q):
    def worker():
        """Processor for the work queue."""
        while True:
            try:
                # Pull an item from the queue
                item = q.get()

                print("Processing a work item.")
                # Run it
                item()

                # Tell the queue that a task is done
                q.task_done()

            except ThreadStop:
                q.put(item)
                break

            except Exception:
                logging.exception("Failure during worker...")
                continue

    return worker


@contextmanager
def worker_queue(worker_count):
    """Initialize the worker queue and pool."""
    q = Queue()

    # Create and start the desired number of worker threads
    for i in range(worker_count):
        t = Thread(target=create_worker(q))
        t.daemon = True
        t.start()

    # Yield control of the queue back to the usage site
    yield q

    # OK, let's clean up.

    # Wait for the workers.
    q.join()

    # Tell the workers to go home.
    q.put(stop_queue)