Bi-directional communication over Popen

0_README.md

Bi-directional Communication over Popen

This gist illustrates how two processes can exchange information via subprocess.Popen.

Goal

Integrate an externally running user interface, potentially written in another language. For example, a GUI written in PyQt5 and Python 3 running inside of Autodesk Maya (Python 2.7, PySide).

Description

child.py is run via parent.py and listens for input over stdin and sends requests to the parent process via stdout. The parent then emits data via the child-processes stdin and listens on it's stdout.

 __________                _________
|          |              |         |
|          |--- stdin ---->         |
|  Parent  |              |  Child  |
|          <--- stdout ---|         |
|__________|              |_________|

Usage

You'll need PyQt5 and Python 2.x in order to run this example.

$ git clone https://gist.github.com/a779958135b3659d639a.git bidir && cd bidir
$ python parent.py

Topics for conversation

There are two topics of communication on both ends.

• request: A process has transmitted output
• response: A process has produced output

Topics are distinguished by each line of communication being prefixed with either request: or response:. That way, both will know whether the incoming data is meant as a response to an earlier request, or as a request for something to do on behalf of the other.

The parent listens for requests only, but it's a good idea to listen for both in case the child has trouble performing a request and wishes to communicate an error of some kind.

Ths child listens on both requests and responses. Responses to requests transmitted to the parent are coming in via the child process's stdin, whereas requests are made through its stdout.

Things to note

1. A process won't output anything new unless the given stdin command ends with a newline
2. Synchronous request/reply is difficult, as they are both continously listen for input without considering where one "stream" begins or ends. It's arguable however whether one should bother designing for synchronousy when it comes to user interfaces as opposed to designing for asynchronousy. In this example, update() is sent from the child to the parent, but the child doesn't wait for input. Instead, the parent sends it whenever it can (without blocking itself either) and the child merely responds to the input. In that sense, it didn't matter that the child was the one who originally made the request.

child.py

import sys
import threading
import contextlib

from PyQt5 import QtWidgets, QtCore


class Child(QtWidgets.QWidget):
    appended = QtCore.pyqtSignal(str)
    resetted = QtCore.pyqtSignal()

    def __init__(self, parent=None):
        super(Child, self).__init__(parent)
        self.setWindowTitle("Child")
        self.setFixedSize(500, 300)

        output = QtWidgets.QPlainTextEdit()
        output.setReadOnly(True)
        button = QtWidgets.QPushButton("Update")
        button.pressed.connect(self.request_update)

        layout = QtWidgets.QVBoxLayout(self)
        layout.addWidget(output)
        layout.addWidget(button)

        self.thread = threading.Thread(target=self.listen).start()
        self.output = output

        self.appended.connect(self.on_append)
        self.resetted.connect(self.on_reset)

    def request_update(self):
        """Ask for items from parent"""
        self.on_append("Querying..")
        self.send("request:update")

    def update(self, items):
        self.appended.emit("Updating with %s" % str(items))
        self.resetted.emit()

    def on_reset(self):
        self.setStyleSheet("")

    def run(self, command, args):
        """Run command from parent

        Arguments:
            command (str): Command to run
            args (list): Arguments passed to command

        """

        self.appended.emit("> %s %s" % (command, " ".join(args)))

        if command == "changeColor":
            if not args:
                return self.appended.emit("Expected color in arguments")

            self.setStyleSheet("""

            QPlainTextEdit {
                background-color: %s;
            }

            """ % args[0])

            self.appended.emit("Color changed to \"%s\"" % args[0])

        else:
            sys.stderr.write("Command is: %r" % command)
            self.appended.emit("Unknown command: \"%s\"" % command)

    def listen(self):
        while True:
            line = sys.stdin.readline()
            if not line:
                break

            # Every line ends with a newline and on
            # Windows a newline + carriage return
            line = line.rstrip()

            topic, content = line.split(":", 1)

            if topic == "response":
                command, value = content.split("|")

                if command == "update":
                    self.update(value.split("\\"))

                else:
                    self.appended.emit("Unknown response: \"%s\"" % command)

            elif topic == "request":
                command, args = (content.split(" ", 1) + [None])[:2]
                self.run(command, args.split() if args else [])

            else:
                self.appended.emit("Unknown input: \"%s\"" % line.rstrip())

    def on_append(self, line):
        self.output.appendPlainText(line)

    def send(self, line):
        sys.stdout.write(line + "\n")


@contextlib.contextmanager
def application():
    app = QtWidgets.QApplication(sys.argv)
    yield
    app.exec_()


if __name__ == '__main__':
    with application():
        window = Child()
        window.show()

parent.py

import sys
import time
import Queue
import threading
import subprocess
import contextlib

from PyQt5 import QtWidgets, QtCore


class Parent(QtWidgets.QWidget):
    appended = QtCore.pyqtSignal(str)

    def __init__(self, to_queue, from_queue, parent=None):
        super(Parent, self).__init__(parent)
        self.setWindowTitle("Parent")
        self.setFixedSize(400, 500)

        output = QtWidgets.QPlainTextEdit()
        edit = QtWidgets.QLineEdit()

        output.setReadOnly(True)

        layout = QtWidgets.QVBoxLayout(self)
        layout.addWidget(output)
        layout.addWidget(edit)

        edit.setFocus()

        # Signals
        edit.returnPressed.connect(self.send)
        self.appended.connect(output.appendPlainText)

        # References
        self.edit = edit
        self.output = output
        self.to_queue = to_queue
        self.from_queue = from_queue

        threading.Thread(target=self.listen).start()

    def send(self):
        command = self.edit.text()

        if not command:
            return

        self.to_queue.put("request:" + command)
        self.edit.clear()

    def run(self, command):
        if command == "update":
            self.appended.emit("Update requested..")
            time.sleep(1)

            result = list()
            for i in range(10):
                result.append("%d" % i)

            self.appended.emit("Sending items..")
            self.to_queue.put("response:update|" + "\\".join(result))
            self.appended.emit("Items sent successfully")

    def listen(self):
        for line in iter(self.from_queue.get, None):
            self.appended.emit("> %s" % line)

            topic, contents = line.split(":", 1)

            if topic == "request":
                self.run(contents)


def writer(f, queue):
    """Send to file-like object

    Arguments:
        f (file): Channel through which to send
        queue (Queue.Queue): Plain-text to send

    """

    for line in iter(queue.get, None):
        f.write(line + "\n")
    f.close()


def reader(f, queue):
    """Read from file-like object

    Arguments:
        f (file): Channel from which to read
        queue (Queue): Store output here

    """

    for line in iter(f.readline, b""):
        queue.put(line.rstrip())
    f.close()


@contextlib.contextmanager
def application():
    app = QtWidgets.QApplication(sys.argv)
    yield
    app.exec_()


if __name__ == '__main__':
    process = subprocess.Popen(["python", "-u", "child.py"],
                               stdout=subprocess.PIPE,
                               stdin=subprocess.PIPE)
    to_queue = Queue.Queue()
    from_queue = Queue.Queue()

    threading.Thread(target=writer, args=[process.stdin, to_queue]).start()
    threading.Thread(target=reader, args=[process.stdout, from_queue]).start()

    with application():
        window = Parent(to_queue, from_queue)
        window.show()

    # Cleanup threads
    to_queue.put(None)
    from_queue.put(None)