Multiprocessing in Python with Custom Workers

custom_worker_processes.py

import random
import multiprocessing


context = multiprocessing.get_context()


class Worker(context.Process):

    def __init__(self, *args, **kwargs):
        """ Constructor.

        :return void
        """
        super().__init__(*args, **kwargs)
        print('Custom worker constructed.')

        # Assign any attributes to the worker...
        self.attribute_on_worker = random.randrange(0, 1000)

    @classmethod
    def register(cls, context):
        """ Ensure this worker is used to process
        the tasks.
        
        :return void
        """
        context.Process = cls

def work(x):
    """ The actual work to be done.
    
    :return void
    """
    p = multiprocessing.current_process()
    print("{} is doing this work. The worker has this attribute: {}.".format(
        p.name, p.attribute_on_worker
    ))

def error_callback(e):
    """ A function to raise exceptions that may occur
    in the worker.

    :raises Exception
    :return void
    """
    raise e


Worker.register(context)
pool = context.Pool(4)

# Add the work into the pool.
for i in range(10):
    pool.apply_async(
        work,
        (i,),
        error_callback = error_callback
    )

pool.close()
pool.join()


"""
Output:

>    Custom worker constructed.
>    Custom worker constructed.
>    Custom worker constructed.
>    Custom worker constructed.
>    Worker-1 is doing this work. The worker has this attribute: 554.
>    Worker-2 is doing this work. The worker has this attribute: 523.
>    Worker-3 is doing this work. The worker has this attribute: 300.
>    Worker-1 is doing this work. The worker has this attribute: 554.
>    Worker-2 is doing this work. The worker has this attribute: 523.
>    Worker-3 is doing this work. The worker has this attribute: 300.
>    Worker-1 is doing this work. The worker has this attribute: 554.
>    Worker-2 is doing this work. The worker has this attribute: 523.
>    Worker-3 is doing this work. The worker has this attribute: 300.
>    Worker-4 is doing this work. The worker has this attribute: 457.
"""