Skip to content

Instantly share code, notes, and snippets.

@dhilst
Last active December 6, 2017 03:12
Show Gist options
  • Save dhilst/e786ad5c549e4120a3860443d3aba4d2 to your computer and use it in GitHub Desktop.
Save dhilst/e786ad5c549e4120a3860443d3aba4d2 to your computer and use it in GitHub Desktop.
Python worker pool that doesn't waste memory.
#!/usr/bin/env python3
import sys
import multiprocessing as mp
import subprocess as sp
import shlex as sh
from itertools import cycle
def sh(cmd):
return sp.Popen(cmd, shell=True, stdout=sys.stdout, stderr=sys.stderr).wait()
q = mp.Queue(mp.cpu_count())
def echo(message):
return sh('echo {} && sleep 1'.format(message))
def exec_echo(q):
while True:
msg = q.get()
echo(msg)
cpu = mp.cpu_count()
queue = mp.Queue(cpu)
for c in range(cpu):
p = mp.Process(target=exec_echo, args=(q,))
p.daemon = True
p.start()
while True:
q.put('Hello')
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment