Last active
November 2, 2018 21:59
-
-
Save darksinge/93e021f069a5c4d82434080e403fd347 to your computer and use it in GitHub Desktop.
Example of using the multiprocessing module in Python to fully utilize a CPU with 'n' cores.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import multiprocessing | |
from threading import Thread | |
import random | |
import string | |
class ThreadWorker(Thread): | |
def __init__(self, *args, **kwargs): | |
super().__init__(*args, **kwargs) | |
def run(self): | |
target = getattr(self, '_target', None) | |
args = getattr(self, '_args', None) | |
if not all([target, args]): | |
raise AttributeError("thread has no attribute 'target' and/or 'args'") | |
sum_ = 0 | |
for arg in args: | |
sum_ += target(arg) | |
# return sum_ | |
def do_math(x): | |
return x*x | |
def ptarget(queue, t_count): | |
while True: | |
args = queue.get() | |
if args: | |
workers = [] | |
for i in range(0, t_count): | |
randargs = [random.randint(0, 100) for _ in range(0, 1000)] | |
worker = ThreadWorker(args=list(randargs), target=do_math) | |
worker.daemon = True | |
workers.append(worker) | |
for w in workers: | |
w.start() | |
for w in workers: | |
w.join() | |
queue.task_done() | |
def init_procs(processes: int = 1, threads: int = 1): | |
q = multiprocessing.JoinableQueue() | |
# create processes | |
procs = [None] * processes | |
for i in range(0, processes): | |
p = multiprocessing.Process(target=ptarget, args=(q, threads), name=string.ascii_letters[i]) | |
p.daemon = True | |
procs[i] = p | |
# start each process | |
for p in procs: | |
p.start() | |
return q | |
if __name__ == '__main__': | |
queue = init_procs(8, 16) | |
while True: | |
queue.put(random.randint(1, 100)) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment