diff --git a/src/log_plotter/log_parser.py b/src/log_plotter/log_parser.py index 9afbcaa..9cf877a 100644 --- a/src/log_plotter/log_parser.py +++ b/src/log_plotter/log_parser.py @@ -4,9 +4,11 @@ import metayaml import multiprocessing from functools import reduce -from log_plotter.plot_utils import readOneTopic, replaceRHString +from log_plotter.plot_utils import readOneTopic, readOneTopicZip, readOneTopicTar, replaceRHString from log_plotter.graph_legend import expand_str_to_list - +import zipfile +import tarfile +import os.path class LogParser(object): @@ -41,6 +43,28 @@ def __init__(self, fname, plot_conf, layout_conf, read_yaml = True, start_idx = self.start_idx = start_idx self.data_length = data_length + self.zip_file = None + self.tar_file = None + try: + if zipfile.is_zipfile(fname): + ##self.zip_file = zipfile.ZipFile(fname) + self.zip_file = fname + root_, ext_ = os.path.splitext(fname) + self.fname = os.path.basename(root_) + print('Open zip file: %s %s'%(self.zip_file, self.fname)) + elif tarfile.is_tarfile(fname): + ##self.tar_file = tarfile.TarFile(fname) + self.tar_file = fname + root_, ext_ = os.path.splitext(fname) + self.fname = os.path.basename(root_) + ## hotfix for with statement + tarfile.ExFileObject.__enter__ = lambda self_: self_ + tarfile.ExFileObject.__exit__ = lambda self_, a, b, c: None + print('Open tar file: %s %s'%(self.tar_file, self.fname)) + except Exception as e: + ## is_tarfile throw error + pass + def readData(self): ''' read log data from log files and store dataListDict @@ -59,9 +83,17 @@ def readData(self): # store data in parallel fname_list = replaceRHString([self.fname + '.' + ext for ext in topic_list]) - fname_list = map(lambda fn: [fn, self.start_idx, self.data_length], fname_list) ## add start_idx, data_length + read_func = readOneTopic + if not self.zip_file is None: + fname_list = map(lambda fn: [fn, self.start_idx, self.data_length, self.zip_file], fname_list) ## add start_idx, data_length + read_func = readOneTopicZip + elif not self.tar_file is None: + fname_list = map(lambda fn: [fn, self.start_idx, self.data_length, self.tar_file], fname_list) ## add start_idx, data_length + read_func = readOneTopicTar + else: + fname_list = map(lambda fn: [fn, self.start_idx, self.data_length], fname_list) ## add start_idx, data_length pl = multiprocessing.Pool() - data_list = pl.map(readOneTopic, fname_list) + data_list = pl.map(read_func, fname_list) for topic, data in zip(topic_list, data_list): self.dataListDict[topic] = data # set the fastest time as 0 diff --git a/src/log_plotter/plot_utils.py b/src/log_plotter/plot_utils.py index 437e098..80b7076 100644 --- a/src/log_plotter/plot_utils.py +++ b/src/log_plotter/plot_utils.py @@ -7,6 +7,8 @@ import fnmatch import numpy import functools +import zipfile +import tarfile def my_time(func): """ @@ -23,19 +25,20 @@ def wrapper(*args, **kwargs): return wrapper # seems that we should declare global function for multiprocess -def readOneTopic(args): +def readOneTopic(args, open_func = lambda fname: open(fname, 'r')): fname = args[0] start = 0 length = 0 - if len(args) > 1: + arglen = len(args) + if arglen > 1: start = args[1] - if len(args) > 2: + if arglen > 2: length = args[2] data = [] - cntr = 0 endidx = start + length try: - with open(fname, 'r') as f: + with open_func(fname) as f: + cntr = 0 reader = csv.reader(f, delimiter=' ') for row in reader: try: @@ -56,6 +59,18 @@ def readOneTopic(args): return None return numpy.array(data) +def readOneTopicZip(args): + if len(args) > 3: + name_ = args[3] + obj_ = zipfile.ZipFile(name_) + return readOneTopic(args, open_func = lambda fname: obj_.open(fname, 'r')) + +def readOneTopicTar(args): + if len(args) > 3: + name_ = args[3] + obj_ = tarfile.TarFile(name_) + return readOneTopic(args, open_func = lambda fname: obj_.extractfile(fname)) + def findFile(pattern, path): """ Return file list matching specific substring