Combine and synchronize stdout and stderr?

Question

Let's say I run exe from a python script using:

subprocess.call(cmdArgs,stdout=outf, stderr=errf)

when outf and errf are text file descriptors.

Is there a way that I can generate a combined and synchronized text file of both stdout and stderr on top of it? It must be formatted with time and source (ours / err).

thanks

Answer 1

This is a bit tricky since you need to poll the file descriptors of the stdout and stderr subprocess while it is running to get the exact timestamps. You also need to cut the output to a list of strings so that the final results can be easily combined and sorted. You can easily combine the two streams as you read them, but this was not part of the question.

I wrote it quickly, but it could be made cleaner and more compact:

 import datetime import os import select import subprocess class Stream(object): def __init__(self, name, impl): self._name = name self._impl = impl self._buf = '' self._rows = [] def fileno(self): "Pass-through for file descriptor." return self._impl.fileno() def read(self, drain=0): "Read from the file descriptor. If 'drain' set, read until EOF." while self._read() is not None: if not drain: break def _read(self): "Read from the file descriptor" fd = self.fileno() buf = os.read(fd, 4096) if not buf: return None if '\n' not in buf: self._buf += buf return [] # prepend any data previously read, then split into lines and format buf = self._buf + buf tmp, rest = buf.rsplit('\n', 1) self._buf = rest now = datetime.datetime.now().isoformat() rows = tmp.split('\n') self._rows += [(now, '%s %s: %s' % (self._name, now, r)) for r in rows] def run(cmd, timeout=0.1): """ Run a command, read stdout and stderr, prefix with timestamp, and return a dict containing stdout, stderr and merged. """ PIPE = subprocess.PIPE proc = subprocess.Popen(cmd, stdout=PIPE, stderr=PIPE) streams = [ Stream('stdout', proc.stdout), Stream('stderr', proc.stderr) ] def _process(drain=0): res = select.select(streams, [], [], timeout) for stream in res[0]: stream.read(drain) while proc.returncode is None: proc.poll() _process() _process(drain=1) # collect results, merge and return result = {} temp = [] for stream in streams: rows = stream._rows temp += rows result[stream._name] = [r[1] for r in rows] temp.sort() result['merged'] = [r[1] for r in temp] return result res = run(['ls', '-l', '.', 'xyzabc']) for key in ('stdout', 'stderr', 'merged'): print print '\n'.join(res[key]) print '-'*40 

Output Example:

 stdout 2011-03-03T19:30:44.838145: .: stdout 2011-03-03T19:30:44.838145: total 0 stdout 2011-03-03T19:30:44.838338: -rw-r--r-- 1 pat pat 0 2011-03-03 19:30 bar stdout 2011-03-03T19:30:44.838518: -rw-r--r-- 1 pat pat 0 2011-03-03 19:30 foo ---------------------------------------- stderr 2011-03-03T19:30:44.837189: ls: cannot access xyzabc: No such file or directory ---------------------------------------- stderr 2011-03-03T19:30:44.837189: ls: cannot access xyzabc: No such file or directory stdout 2011-03-03T19:30:44.838145: .: stdout 2011-03-03T19:30:44.838145: total 0 stdout 2011-03-03T19:30:44.838338: -rw-r--r-- 1 pat pat 0 2011-03-03 19:30 bar stdout 2011-03-03T19:30:44.838518: -rw-r--r-- 1 pat pat 0 2011-03-03 19:30 foo ---------------------------------------- 

Answer 2

You can combine them with passing subprocess.STDOUT as the stderr argument to subprocess.Popen , but I don’t know if they will be formatted with time and source.