Executing Python from Go

main.go

package main

import (
    "syscall"
    "log"
    "bufio"
    "io/ioutil"
    "os/exec"
    msgpack "gopkg.in/vmihailenco/msgpack.v2"
)

type Command struct {
    Module string
    Function string
    Args []interface{}
    // TODO
    // Kwargs
}

type Worker struct {
    In chan Command
    Out chan interface{}
    Worker *exec.Cmd
}

func PyInit() Worker {
    input := make(chan Command, 10)
    output := make(chan interface{}, 10)
    worker := exec.Command("python", "./worker.py")

    go func(){
        stdin, err := worker.StdinPipe()
        if err != nil {
            log.Fatalf("Error obtaining stdin: %s", err.Error())
        }
        stdout, err := worker.StdoutPipe()
        if err != nil {
            log.Fatalf("Error obtaining stdout: %s", err.Error())
        }
        stderr, err := worker.StderrPipe()
        if err != nil {
            log.Fatalf("Error obtaining stderr: %s", err.Error())
        }
        log.Print("=> starting")
        err = worker.Start()
        if err != nil {
            log.Fatal(err)
        }
        reader := bufio.NewReader(stdout)

        go func(){
            for cmd := range input {
                payload, err := msgpack.Marshal(cmd)
                if err != nil {
                    panic(err)
                }
                stdin.Write(payload)
                stdin.Write([]byte("\n"))

                var response []byte

                for {
                    line, _, _ := reader.ReadLine()
                    if len(line) == 0 {
                        return
                    }
                    // TODO: more reliable way
                    if line[0] == '-' && line[1] == '-'{
                        break
                    }
                    response = append(response, line...)
                    response = append(response, []byte("\n")...)
                }
                var v interface{}
                err = msgpack.Unmarshal(response, &v)
                if err != nil {
                    panic(err)
                }
                output <- v
            }
        }()

        log.Print("=> wait terminating")
        stderr_out, _ := ioutil.ReadAll(stderr)
        err = worker.Wait()
        if err != nil {
            log.Printf("=> unexpected terminating\n%s", stderr_out)
            log.Fatal(err)
        }
    }()

    return Worker{In: input, Out: output, Worker: worker}
}


func PyCall(worker Worker, cmd Command) interface{} {
    worker.In <- cmd
    result := <- worker.Out
    return result
}

func main() {
    worker := PyInit()

    result1 := PyCall(
        worker,
        Command{
            Module: "sys",
            Function: "version.__str__"})
    log.Printf("Result1: %s", result1)

    result2 := PyCall(
        worker,
        Command{
            Module: "builtins",
            Function: "str.join",
            Args: []interface{}{",", []string{"a", "b"}}})
    log.Printf("Result2: %s", result2)

    log.Printf("PID %s", worker.Worker.Process.Pid)
    if err := syscall.Kill(worker.Worker.Process.Pid, syscall.SIGINT); err != nil {
        log.Println("failed to kill: ", err)
    }
}

Makefile

init:
	pip install msgpack-python==0.4.8
	go get gopkg.in/vmihailenco/msgpack.v2

run:
	go run main.go

worker.py

import os
import sys
import importlib
import msgpack

import logging
logging.basicConfig(filename='worker.log', level=logging.DEBUG)

class Command:

    __slots__ = ('module', 'function', 'args', 'kwargs')

    def __init__(self, Module, Function, Args=None, Kwargs=None):
        self.module = Module
        self.function = Function
        self.args = Args or []
        self.kwargs = Kwargs or {}

    def execute(self):
        if self.module not in sys.modules:
            importlib.import_module(self.module)
        objects = self.function.split(".")

        fn = sys.modules[self.module]
        for o in objects:
            fn = getattr(fn, o)
        return fn(*self.args, **self.kwargs)

    def __repr__(self):
        return f'<Command {self.module}:{self.function}>'


def run():
    while True:
        logging.info('start reading...')

        data = sys.stdin.buffer.readline()
        logging.info('Input: %s' % data)
        if not data:
            continue

        decoded_data = msgpack.unpackb(data[:-1], encoding='utf-8')
        command = Command(
            **decoded_data
        )
        logging.info('Cmd: %s' % command)

        result = command.execute()
        logging.info('Result: %s' % result)

        payload = msgpack.packb(result)
        logging.info('Payload: %r' % payload)

        sys.stdout.buffer.write(payload)
        sys.stdout.buffer.write(b"\n")
        sys.stdout.buffer.write(b"--\n")
        sys.stdout.buffer.flush()


def main():
    try:
        run()
    except KeyboardInterrupt:
        logging.info('Interrupt\n')


if __name__ == '__main__':
    main()