add code for reading zipped logfile

src/log_plotter/log_parser.py

@@ -4,9 +4,11 @@
 import metayaml
 import multiprocessing
 from functools import reduce
-from log_plotter.plot_utils import readOneTopic, replaceRHString
+from log_plotter.plot_utils import readOneTopic, readOneTopicZip, readOneTopicTar, replaceRHString
 from log_plotter.graph_legend import expand_str_to_list
-
+import zipfile
+import tarfile
+import os.path
 
 class LogParser(object):
 
@@ -41,6 +43,28 @@ def __init__(self, fname, plot_conf, layout_conf, read_yaml = True, start_idx =
         self.start_idx = start_idx
         self.data_length = data_length
 
+        self.zip_file = None
+        self.tar_file = None
+        try:
+            if zipfile.is_zipfile(fname):
+                ##self.zip_file = zipfile.ZipFile(fname)
+                self.zip_file = fname
+                root_, ext_ = os.path.splitext(fname)
+                self.fname = os.path.basename(root_)
+                print('Open zip file: %s %s'%(self.zip_file, self.fname))
+            elif tarfile.is_tarfile(fname):
+                ##self.tar_file = tarfile.TarFile(fname)
+                self.tar_file = fname
+                root_, ext_ = os.path.splitext(fname)
+                self.fname = os.path.basename(root_)
+                ## hotfix for with statement
+                tarfile.ExFileObject.__enter__ = lambda self_: self_
+                tarfile.ExFileObject.__exit__ = lambda self_, a, b, c: None
+                print('Open tar file: %s %s'%(self.tar_file, self.fname))
+        except Exception as e:
+            ## is_tarfile throw error
+            pass
+
     def readData(self):
         '''
         read log data from log files and store dataListDict
@@ -59,9 +83,17 @@ def readData(self):
 
         # store data in parallel
         fname_list = replaceRHString([self.fname + '.' + ext for ext in topic_list])
-        fname_list = map(lambda fn: [fn, self.start_idx, self.data_length], fname_list) ## add start_idx, data_length
+        read_func = readOneTopic
+        if not self.zip_file is None:
+            fname_list = map(lambda fn: [fn, self.start_idx, self.data_length, self.zip_file], fname_list) ## add start_idx, data_length
+            read_func = readOneTopicZip
+        elif not self.tar_file is None:
+            fname_list = map(lambda fn: [fn, self.start_idx, self.data_length, self.tar_file], fname_list) ## add start_idx, data_length
+            read_func = readOneTopicTar
+        else:
+            fname_list = map(lambda fn: [fn, self.start_idx, self.data_length], fname_list) ## add start_idx, data_length
         pl = multiprocessing.Pool()
-        data_list = pl.map(readOneTopic, fname_list)
+        data_list = pl.map(read_func, fname_list)
         for topic, data in zip(topic_list, data_list):
             self.dataListDict[topic] = data
         # set the fastest time as 0

src/log_plotter/plot_utils.py

@@ -7,6 +7,8 @@
 import fnmatch
 import numpy
 import functools
+import zipfile
+import tarfile
 
 def my_time(func):
     """
@@ -23,19 +25,20 @@ def wrapper(*args, **kwargs):
     return wrapper
 
 # seems that we should declare global function for multiprocess
-def readOneTopic(args):
+def readOneTopic(args, open_func = lambda fname: open(fname, 'r')):
     fname = args[0]
     start = 0
     length = 0
-    if len(args) > 1:
+    arglen = len(args)
+    if arglen > 1:
         start = args[1]
-    if len(args) > 2:
+    if arglen > 2:
         length = args[2]
     data = []
-    cntr = 0
     endidx = start + length
     try:
-        with open(fname, 'r') as f:
+        with open_func(fname) as f:
+            cntr = 0
             reader = csv.reader(f, delimiter=' ')
             for row in reader:
                 try:
@@ -56,6 +59,18 @@ def readOneTopic(args):
         return None
     return numpy.array(data)
 
+def readOneTopicZip(args):
+    if len(args) > 3:
+        name_ = args[3]
+    obj_ = zipfile.ZipFile(name_)
+    return readOneTopic(args, open_func = lambda fname: obj_.open(fname, 'r'))
+
+def readOneTopicTar(args):
+    if len(args) > 3:
+        name_ = args[3]
+    obj_ = tarfile.TarFile(name_)
+    return readOneTopic(args, open_func = lambda fname: obj_.extractfile(fname))
+
 def findFile(pattern, path):
     """
     Return file list matching specific substring