Compact (archived) old log files in python

Question

I use the standard log library in Python. There is a RotatingFileHandler that can, for example, rotate log files.

But he just renames them. It will be great if it can not only rename, but also put old files in the zip archive (or gz, bzip, etc.).

Is there an easy way to achieve this?

Answer 1

I think your best option is to extend the RotatingFileHandler with something like this ( not verified ):

import os from logging.handlers import RotatingFileHandler COMPRESSION_SUPPORTED = {} try: import gzip COMPRESSION_SUPPORTED['gz'] = gzip except ImportError: pass try: import zipfile COMPRESSION_SUPPORTED['zip'] = zipfile except ImportError: pass class NewRotatingFileHandler(RotatingFileHandler): def __init__(self, *args, **kws): compress_mode = kws.pop('compress_mode') try: self.compress_cls = COMPRESSION_SUPPORTED[compress_mode] except KeyError: raise ValueError('"%s" compression method not supported.' % compress_mode) super(NewRotatingFileHandler, self).__init__(self, *args, **kws) def doRollover(self): super(NewRotatingFileHandler, self).doRollover() # Compress the old log. old_log = self.baseFilename + ".1" with open(old_log) as log: with self.compress_cls.open(old_log + '.gz', 'wb') as comp_log: comp_log.writelines(log) os.remove(old_log) 

Answer 2

The accepted answer will only archive 1 file- (basefile.log.1). Other files are not archived. This code will archive all log files except the main file.

 import os import gzip import logging.handlers class NewRotatingFileHandler(logging.handlers.RotatingFileHandler): def __init__(self, filename, **kws): backupCount = kws.get('backupCount', 0) self.backup_count = backupCount logging.handlers.RotatingFileHandler.__init__(self, filename, **kws) def doArchive(self, old_log): with open(old_log) as log: with gzip.open(old_log + '.gz', 'wb') as comp_log: comp_log.writelines(log) os.remove(old_log) def doRollover(self): if self.stream: self.stream.close() self.stream = None if self.backup_count > 0: for i in range(self.backup_count - 1, 0, -1): sfn = "%s.%d.gz" % (self.baseFilename, i) dfn = "%s.%d.gz" % (self.baseFilename, i + 1) if os.path.exists(sfn): if os.path.exists(dfn): os.remove(dfn) os.rename(sfn, dfn) dfn = self.baseFilename + ".1" if os.path.exists(dfn): os.remove(dfn) if os.path.exists(self.baseFilename): os.rename(self.baseFilename, dfn) self.doArchive(dfn) if not self.delay: self.stream = self._open() 

Answer 3

You can automatically write compressed bz2 log files by initializing the RotatingFileHandler with encoding='bz2-codec' :

 import logging import logging.handlers as handlers if __name__=='__main__': log_filename='log_rotate.bz2' logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) handler = handlers.RotatingFileHandler( log_filename, maxBytes=20, backupCount=5, encoding='bz2-codec') logger.addHandler(handler) for i in range(20): logger.debug('i = %d' % i) 

PS. Python3 removed 'bz2-codec' from the set of valid encodings, so this solution is specific to Python2.