Last active
December 6, 2017 03:12
-
-
Save dhilst/e786ad5c549e4120a3860443d3aba4d2 to your computer and use it in GitHub Desktop.
Python worker pool that doesn't waste memory.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python3 | |
import sys | |
import multiprocessing as mp | |
import subprocess as sp | |
import shlex as sh | |
from itertools import cycle | |
def sh(cmd): | |
return sp.Popen(cmd, shell=True, stdout=sys.stdout, stderr=sys.stderr).wait() | |
q = mp.Queue(mp.cpu_count()) | |
def echo(message): | |
return sh('echo {} && sleep 1'.format(message)) | |
def exec_echo(q): | |
while True: | |
msg = q.get() | |
echo(msg) | |
cpu = mp.cpu_count() | |
queue = mp.Queue(cpu) | |
for c in range(cpu): | |
p = mp.Process(target=exec_echo, args=(q,)) | |
p.daemon = True | |
p.start() | |
while True: | |
q.put('Hello') |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment