Python subprocess logging to logger from stdout/stderr

logging_subprocess.py

#!/usr/local/bin/python3

import logging, select, subprocess

LOG_FILE = "test.log"
logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO,filename=LOG_FILE,format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')

def logging_call(popenargs, **kwargs):
    process = subprocess.Popen(popenargs, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)

    def check_io():
           while True:
                output = process.stdout.readline().decode()
                if output:
                    logger.log(logging.INFO, output)
                else:
                    break

    # keep checking stdout/stderr until the child exits
    while process.poll() is None:
        check_io()

def main():
    logging_call(["ls", "-l"])

if __name__ == "__main__":
    try:
        main()
    except KeyboardInterrupt:
        logging.warning("Stopping...")

Whats the best practice here? Particularly for long running processes?

Thank you for the code snippet. Works beautifully for most part except when the process terminates immediately. In this case, only the first line is printed. A small tweak in your code helps:

     def logs(read_till_end=False):
            ready_to_read = select.select([self.child.stdout, self.child.stderr], [], [], 1000)[0]
            for io in ready_to_read:
                line = 'gibberish'
                while line:
                    line = io.readline()
                    if line:
                        self.logger.log(log_level[io], line[:-1])
                        if not read_till_end:
                            break
    
        # keep checking stdout/stderr until the child exits
        while self.child.poll() is None:
            logs()
    
        logs(read_till_end=True)  # check again to catch anything after the process exits
        self.child.wait() # wait for the child process to terminate

Using a read_till_end parameter, I can instruct the function to read till the end of the stream.

Hi 👋

I've done a rewrite from scratch:

• Using py 3.7+ text kwarg so it works on Windows special chars as well
• Using two threads in order to handle stdout and stderr separately i.e. logging.INFO and logging.ERROR

https://gist.github.com/ddelange/6517e3267fb74eeee804e3b1490b1c1d