run_curls.py

import subprocess
import sys
from queue import Queue
from subprocess import Popen
from threading import Thread


curls = Queue(10)

def do_curl():
    print(f'start poller')
    while True:
        i, curl = curls.get()
        proc = Popen(curl.strip(),
                     stdout=subprocess.PIPE,
                     stderr=subprocess.PIPE,
                     shell=True)
        print(f'#{i}#\t[{proc.pid}]:\tIn progress...')
        try:
            outs, errs = proc.communicate(timeout=30)
        except subprocess.TimeoutExpired:
            proc.kill()
            outs, errs = proc.communicate()
        code = proc.wait()
        if code == 0:
            print(f'#{i}#\t[{proc.pid}]\tCurl done: {code} {errs.decode(sys.stdin.encoding)}.')
        else:
            print(f'#{i}#\t[{proc.pid}]\tCurl failed: {code} {errs.decode(sys.stdin.encoding)}\n\t{curl}.')
        curls.task_done()

for i in range(10):
    th = Thread(target=do_curl, daemon=True)
    th.start()

i = 0
for _ in range(100):
    with open("run_curls.txt") as fp:
        lines = fp.readlines()
        for line in lines:
            curls.put((i, line))
            print(f'#{i}#\tCurl enqueued.')
            i += 1

print(f'Join.')
curls.join()