Last active
September 22, 2022 11:05
-
-
Save internetimagery/b1b86cd2502aeafca8c236e235561534 to your computer and use it in GitHub Desktop.
Simple multiprocess with gevent
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from multiprocessing import Pool as MPool, current_process | |
from gevent.threadpool import ThreadPool | |
from gevent.pool import Pool as GPool | |
from contextlib import closing | |
class ProcessPool(ThreadPool): | |
def __init__(self, procs, **kwargs): | |
super(ProcessPool, self).__init__(procs) | |
self._m_pool = MPool(procs, **kwargs) | |
def spawn(self, func, *args, **kwargs): | |
return super(ProcessPool, self).spawn( | |
lambda: self._m_pool.apply(func, args, kwargs) | |
) | |
def close(self): | |
self._m_pool.close() | |
self.join() | |
self.kill() | |
def on_proc(value): | |
print("ON PROC", value, current_process()) | |
return value * 3 | |
def work(value): | |
print("VAL", value) | |
new_value = pool.apply(on_proc, (value,)) | |
print("FROM PROC", new_value) | |
return new_value + 3 | |
if __name__ == "__main__": | |
with closing(ProcessPool(3)) as pool: | |
for val in GPool().map(work, range(10)): | |
print("RES", val) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment