Extract from whooey

run_cmd.py

#!/usr/bin/env python

import os
import sys
import shlex
import subprocess
from threading import Thread
try:
    from Queue import Empty, Queue
except ImportError:
    from queue import Empty, Queue  # python 3.x


def enqueue_output(out, q):
    for line in iter(out.readline, b''):
        q.put(line.decode('utf-8'))
    try:
        out.close()
    except IOError:
        pass


def output_monitor_queue(queue, out):
    p = Thread(target=enqueue_output, args=(out, queue))
    p.start()
    return p


def update_from_output_queue(queue, out):
    lines = []
    while True:
        try:
            line = queue.get_nowait()
            lines.append(line)
        except Empty:
            break

    out += ''.join(map(str, lines))
    return out


def main(command_line, abswd):
    cmd = shlex.split(command_line)
    stdout, stderr = '', ''
    proc = subprocess.Popen(
        cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=abscwd, bufsize=0)

    qout, qerr = Queue(), Queue()
    pout = output_monitor_queue(qout, proc.stdout)
    perr = output_monitor_queue(qerr, proc.stderr)

    prev_std = None

    def check_output(stdout, stderr, prev_std):
        # Check for updates from either (non-blocking)
        stdout = update_from_output_queue(qout, stdout)
        stderr = update_from_output_queue(qerr, stderr)
        changed = False

        # If there are changes, update the db
        if (stdout, stderr) != prev_std:
            prev_std = (stdout, stderr)
            changed = True

        return stdout, stderr, prev_std, changed

    # Loop until the process is complete + both stdout/stderr have EOFd
    while proc.poll() is None or pout.is_alive() or perr.is_alive():
        stdout, stderr, prev_std, changed = check_output(stdout, stderr, prev_std)
        if changed:
            yield prev_std

    # Catch any remaining output
    try:
        proc.stdout.flush()
    except ValueError:  # Handle if stdout is closed
        pass
    stdout, stderr, prev_std, changed = check_output(stdout, stderr, prev_std)

    yield (stdout, stderr)


if __name__ == "__main__":
    cmd = "YOU TIME CONSUMED COMMAND LINE JOB"
    cwd = "/path/to/some/work/dir"
    for out, err in main(cmd, cwd):
        print(out, err)