Last active
March 25, 2022 19:54
-
-
Save somma/44d8bb841290a0a0e904 to your computer and use it in GitHub Desktop.
Code illustrates how to pass instance method to multiprocessing and use functools.partial with pool.map() method.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/python | |
# -*- coding:utf-8 -*- | |
""" | |
Idea and code was taken from stackoverflow(). | |
This sample illustrates how to | |
+ how to pass method of instance method | |
to multiprocessing(idea and code was introduced | |
at http://goo.gl/tRHN1D by torek). | |
+ how to use fuctools.partial() with multiprocessing.pool.map() | |
""" | |
import os | |
import multiprocessing as mp | |
import functools | |
def call_it2(instance, name, arg): | |
"indirect caller for instance methods and multiprocessing" | |
return getattr(instance, name)(arg) | |
class Klass(object): | |
""" | |
""" | |
def __init__(self, nobj, workers=mp.cpu_count()): | |
print "Constructor (in pid=%d)..." % os.getpid() | |
self.count = nobj | |
pool = mp.Pool(processes=workers) | |
# with functools.partial(), I can use pool.map() | |
func_call_it = functools.partial(call_it2, self, 'process_obj') | |
results = pool.map(func_call_it, list(i for i in range(nobj))) | |
pool.close() | |
pool.join() | |
print results | |
def __del__(self): | |
self.count -= 1 | |
print "... Destructor (in pid=%d) count=%d" % (os.getpid(), self.count) | |
def process_obj(self, index): | |
print "object %d" % index | |
return "results" | |
def main(): | |
Klass(nobj=8, workers=3) | |
print 'ok' | |
if __name__ == '__main__': | |
main() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment