File descriptor redirections and capture magic

gistfile1.txt

#!/usr/bin/env python
# python 2.7 and >= 3.4 supported

from __future__ import print_function

import sys

if (sys.version_info < (3, 0)):
    assert(sys.version_info >= (2, 7))
else:
    assert(sys.version_info >= (3, 4))

import os
import logging
import subprocess
import tempfile

# I have not found a solution that would allow to both capture the output,
# and also print it to stderr during script execution.
# Maybe something involving os.pipe could work, but recovering at the end is
# enough for my purpose.

tf = tempfile.NamedTemporaryFile( mode = 'w+', prefix = 'stderr.', delete = False )
if (sys.version_info >= (3, 4)):
    # Doesn't work for subprocess anyway, so it being python3 only is not a big deal
    os.set_inheritable(tf.file.fileno(), True)
    assert(os.get_inheritable(tf.file.fileno()))
sys.stderr = tf.file

# Keep hold of stdout, redirect all to stderr
stdout_fd_2 = os.dup(sys.stdout.fileno())
os.dup2(sys.stderr.fileno(), sys.stdout.fileno())

logger = logging.getLogger('test')
logging.getLogger().setLevel(logging.DEBUG)
logging.basicConfig()

print('stdout test')
# print('stderr test', file = sys.stderr)

logger.info('logger.info test (defaults to stderr)')

subprocess.call( "echo shell stdout test", shell = True )

# When using tf, this still pops on the real stderr, despite the fd being marked inheritable ..
# subprocess doesn't honor sys.stderr and goes to sys.__stderr__ directly?
subprocess.call( "echo shell stderr test >&2", shell = True )

# Will have to explicitly set it, minor inconvenience
subprocess.call( "echo shell stderr test no. 2 >&2", stderr = tf, shell = True )

os.write(stdout_fd_2, 'test payload to initial stdout\n'.encode('utf-8'))

if (tf is not None):
    tf.flush()
    tf.seek(0)
    os.write(stdout_fd_2, '== output captured through stderr: ==\n'.encode('utf-8'))
    os.write(stdout_fd_2, tf.read().encode('utf-8'))

I have not found a solution that would allow to both capture the output, and also print it to stderr during script execution.

@TTimo: Sorry, I'm not sure I understand what your goal is here? Are you looking for a Python equivalent of foo 2>&1 | tee some.log >&2 to make the output of foo go to both a log file and stderr?

If so, it would look something like either this:

tee = subprocess.Popen(['tee', 'some.log'], stdin=subprocess.PIPE, stdout=sys.stderr, stderr=subprocess.STDOUT)
subprocess.call(['foo'], stdin=subprocess.DEVNULL, stdout=tee.stdin, stderr=subprocess.STDOUT)
tee.wait()

or using a Python thread to implement the equivalent of tee.