How can I multiplex output to an OS file descriptor in Python?

Question

The mechanism subprocess.Popenuses a basic file descriptor instead of a file-like object to write it to stdout/stderr. I need to capture both stdout, and stderrwhile they are displayed on the console.

How to create a file descriptor that Popen can use will allow me to do this?

Answer 1

Just a little context: subprocess uses raw file descriptors for the stdin, stdout, stderr objects that you specify because it passes them to POSIX . If you use subprocess.PIPEthen it will create a new channel with os.pipe(). It is also Popen.communicateread to the end of the stream, which may be undesirable if you want to transfer data somewhere else.

Since you want to print the output to stdout, I assume that it outputs the text. You will need to use encoding, errorsor universal_newlinesin Popenfor subprocessto process the file as text (see docs ).

import subprocess

p = subprocess.Popen(
    '/usr/bin/whoami',
    stdout=subprocess.PIPE,  # Control stdout
    universal_newlines=True  # Files opened in text mode
)

# Pipe the data somewhere else too, e.g.: a log file
with open('subprocess.log', 'w') as logfile:
    # p.poll() returns the return code when `p` exits
    while p.poll() is None:
        line = p.stdout.readline()
        # one to our stdout (readline includes the \n)
        print(line, end='')
        # one to the logfile
        logfile.write(line)

stderr, , file=sys.stderr print. , stdin, :

subprocess.Popen('/usr/bin/whoami', stdin=sys.stdin, stdout=subprocess.PIPE, ...)

, . , read .

stderr stdout

stdout, stderr, , .
os.set_blocking, , read , . .
, stdout stderr; aysncio module:

import asyncio
import sys

PROCESS_PATH = '/bin/mixed_output'

class MultiplexProtocol(asyncio.SubprocessProtocol):
    def __init__(self, exit_future):
        self.exit_future = exit_future

    def pipe_data_received(self, fd, data):
        if fd == sys.stdout.fileno():
            print(data.decode('utf-8'), file=sys.stdout, end='')
        elif fd == sys.stderr.fileno():
            print(data.decode('utf-8'), file=sys.stderr, end='')

    def process_exited(self):
        self.exit_future.set_result(True)


async def launch_subprocess(loop):
    # Future marking the end of the process
    exit_future = asyncio.Future(loop=loop)
    # Use asyncio subprocess
    create_subp = loop.subprocess_exec(
        lambda: MultiplexProtocol(exit_future),
        PROCESS_PATH,
        stdout=asyncio.subprocess.PIPE,
        stderr=asyncio.subprocess.PIPE,
        stdin=None
    )
    transport, protocol = await create_subp
    await exit_future
    # Close the pipes
    transport.close()


loop = asyncio.get_event_loop()
loop.run_until_complete(launch_subprocess(loop))

, - , MultiplexProtocol.pipe_data_received .