Example showing how to use instance methods with the multiprocessing module

multiprocess_with_instance_methods.py

from multiprocessing import Pool
from functools import partial

def _pickle_method(method):
	func_name = method.im_func.__name__
	obj = method.im_self
	cls = method.im_class
	if func_name.startswith('__') and not func_name.endswith('__'): #deal with mangled names
		cls_name = cls.__name__.lstrip('_')
		func_name = '_' + cls_name + func_name
	return _unpickle_method, (func_name, obj, cls)

def _unpickle_method(func_name, obj, cls):
	for cls in cls.__mro__:
		try:
			func = cls.__dict__[func_name]
		except KeyError:
			pass
		else:
			break
	return func.__get__(obj, cls)

import copy_reg
import types
copy_reg.pickle(types.MethodType, _pickle_method, _unpickle_method)

class someClass(object):
	def __init__(self):
		pass
	
	def f(self, x=None):
		#can put something expensive here to verify CPU utilization
		if x is None: return 99
		return x*x

	def go(self):
		pool = Pool()             
		print pool.map(self.f, range(10))

if __name__=='__main__':
	sc = someClass()
	sc.go()
	x=[someClass(),someClass(),someClass()]
	p=Pool()
	filled_f=partial(someClass.f,x=9)
	print p.map(filled_f,x)
	print p.map(someClass.f,x)

Hi, Bryce. I'm wondering if you could help me with a doubt when applying your code (http://stackoverflow.com/questions/21111106/cant-pickle-multiprocessing-python). Thank you.

Very good and useful. However, it is important to note here that this will work correctly only if the function to parallelize does not use any of the class' mutable members. The reason of this is that the the copies of the class in each process do not share state. Let me illustrate:

class someClass(object):
    def __init__(self):
        self.a = 10

    def f(self, x=None):
        # can put something expensive here to verify CPU utilization
        if x is None: return 99
        self.a = self.a + x
        return self.a

    def go(self):
        pool = Pool()
        print pool.map(self.f, range(10))

    def go_seq(self):
        print map(self.f, range(10))


if __name__ == '__main__':
    sc = someClass()
    sc.go()
    sc = someClass()
    sc.go_seq()

What is the solution to this problem. In my case I have a list as a class variable which the class method updates. Thank you very much.

Either store the state in the local scope and return it out, or use multiprocessing.Value.