Skip to content

Instantly share code, notes, and snippets.

@darksinge
Last active November 2, 2018 21:59
Show Gist options
  • Save darksinge/93e021f069a5c4d82434080e403fd347 to your computer and use it in GitHub Desktop.
Save darksinge/93e021f069a5c4d82434080e403fd347 to your computer and use it in GitHub Desktop.
Example of using the multiprocessing module in Python to fully utilize a CPU with 'n' cores.
import multiprocessing
from threading import Thread
import random
import string
class ThreadWorker(Thread):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def run(self):
target = getattr(self, '_target', None)
args = getattr(self, '_args', None)
if not all([target, args]):
raise AttributeError("thread has no attribute 'target' and/or 'args'")
sum_ = 0
for arg in args:
sum_ += target(arg)
# return sum_
def do_math(x):
return x*x
def ptarget(queue, t_count):
while True:
args = queue.get()
if args:
workers = []
for i in range(0, t_count):
randargs = [random.randint(0, 100) for _ in range(0, 1000)]
worker = ThreadWorker(args=list(randargs), target=do_math)
worker.daemon = True
workers.append(worker)
for w in workers:
w.start()
for w in workers:
w.join()
queue.task_done()
def init_procs(processes: int = 1, threads: int = 1):
q = multiprocessing.JoinableQueue()
# create processes
procs = [None] * processes
for i in range(0, processes):
p = multiprocessing.Process(target=ptarget, args=(q, threads), name=string.ascii_letters[i])
p.daemon = True
procs[i] = p
# start each process
for p in procs:
p.start()
return q
if __name__ == '__main__':
queue = init_procs(8, 16)
while True:
queue.put(random.randint(1, 100))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment