diff --git a/setup.cfg b/setup.cfg index 6b9d2c2c12..9d1c0134e5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -41,6 +41,7 @@ install_requires = httplib2 humanfriendly lupa + lxml ndg-httpsclient netaddr paramiko diff --git a/teuthology/exceptions.py b/teuthology/exceptions.py index cbe8b5941f..da38343545 100644 --- a/teuthology/exceptions.py +++ b/teuthology/exceptions.py @@ -212,3 +212,27 @@ class NoRemoteError(Exception): def __str__(self): return self.message + + +class UnitTestError(Exception): + """ + Exception thrown on unit test failure + """ + def __init__(self, exitstatus=None, node=None, label=None, message=None): + self.exitstatus = exitstatus + self.node = node + self.label = label + self.message = message + + def __str__(self): + prefix = "Unit test failed" + if self.label: + prefix += " ({label})".format(label=self.label) + if self.node: + prefix += " on {node}".format(node=self.node) + if self.exitstatus: + prefix += " with status {status}".format(status=self.exitstatus) + return "{prefix}: '{message}'".format( + prefix=prefix, + message=self.message, + ) diff --git a/teuthology/orchestra/remote.py b/teuthology/orchestra/remote.py index 0392acf87e..ce77a519cf 100644 --- a/teuthology/orchestra/remote.py +++ b/teuthology/orchestra/remote.py @@ -11,7 +11,8 @@ from teuthology.orchestra.opsys import OS import teuthology.provision from teuthology import misc -from teuthology.exceptions import CommandFailedError +from teuthology.exceptions import CommandFailedError, UnitTestError +from teuthology.util.scanner import UnitTestScanner from teuthology.misc import host_shortname import errno import re @@ -523,6 +524,20 @@ def run(self, **kwargs): r.remote = self return r + def run_unit_test(self, xml_path_regex, output_yaml, **kwargs): + try: + r = self.run(**kwargs) + except CommandFailedError as exc: + if xml_path_regex: + error_msg = UnitTestScanner(remote=self).scan_and_write(xml_path_regex, output_yaml) + if error_msg: + raise UnitTestError( + exitstatus=exc.exitstatus, node=exc.node, + label=exc.label, message=error_msg + ) + raise exc + return r + def _sftp_put_file(self, local_path, remote_path): """ Use the paramiko.SFTPClient to put a file. Returns the remote filename. @@ -543,12 +558,14 @@ def _sftp_get_file(self, remote_path, local_path): sftp.get(remote_path, local_path) return local_path - def _sftp_open_file(self, remote_path): + def _sftp_open_file(self, remote_path, mode=None): """ Use the paramiko.SFTPClient to open a file. Returns a paramiko.SFTPFile object. """ sftp = self.ssh.open_sftp() + if mode: + return sftp.open(remote_path, mode) return sftp.open(remote_path) def _sftp_get_size(self, remote_path): diff --git a/teuthology/orchestra/test/xml_files/test_scan_nose.xml b/teuthology/orchestra/test/xml_files/test_scan_nose.xml new file mode 100644 index 0000000000..d0fab48161 --- /dev/null +++ b/teuthology/orchestra/test/xml_files/test_scan_nose.xml @@ -0,0 +1,73 @@ + + + + + +> begin captured logging << -------------------- +botocore.hooks: DEBUG: Event choose-service-name: calling handler +PUT +/test-client.0-2txq2dyjghs0vdf-335 + +host:smithi196.front.sepia.ceph.com +x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +x-amz-date:20220929T065029Z + +host;x-amz-content-sha256;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +botocore.auth: DEBUG: StringToSign: +AWS4-HMAC-SHA256 +20220929T065029Z +20220929/us-east-1/s3/aws4_request +ddfd952c0ac842cff08711f6b1425bec213bd1f69ae5ae6f37afb7a2f66e7fcb +botocore.auth: DEBUG: Signature: +8b7f685e9b8a9a807437088da293390ac21ed9a10acf51903a8da2281bdc9c45 +botocore.hooks: DEBUG: Event request-created.s3.CreateBucket: calling handler +botocore.endpoint: DEBUG: Sending http request: +urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): smithi196.front.sepia.ceph.com:80 +urllib3.connectionpool: DEBUG: http://smithi196.front.sepia.ceph.com:80 "PUT /test-client.0-2txq2dyjghs0vdf-335 HTTP/1.1" 200 0 +botocore.parsers: DEBUG: Response headers: {'x-amz-request-id': 'tx00000e29af2294ab8b56c-0063354035-1157-default', 'Content-Length': '0', 'Date': 'Thu, 29 Sep 2022 06:50:29 GMT', 'Connection': 'Keep-Alive'} +botocore.parsers: DEBUG: Response body: +b'' +botocore.hooks: DEBUG: Event needs-retry.s3.CreateBucket: calling handler +botocore.retryhandler: DEBUG: No retry needed. +GET +/test-client.0-2txq2dyjghs0vdf-335 +tagging= +host:smithi196.front.sepia.ceph.com +x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +x-amz-date:20220929T065029Z + +host;x-amz-content-sha256;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +botocore.auth: DEBUG: StringToSign: +AWS4-HMAC-SHA256 +20220929T065029Z +20220929/us-east-1/s3/aws4_request +8a096d01796a8a6afca50c1bc3bc5c9098917c26a6dba7e752412ce31041c575 +botocore.auth: DEBUG: Signature: +a58a94727b0c0d6d43e8783c91499ce9a9758260aa09a286524c0eb1bc4883d1 +botocore.hooks: DEBUG: Event request-created.s3.GetBucketTagging: calling handler +botocore.endpoint: DEBUG: Sending http request: +urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): smithi196.front.sepia.ceph.com:80 +urllib3.connectionpool: DEBUG: http://smithi196.front.sepia.ceph.com:80 "GET /test-client.0-2txq2dyjghs0vdf-335?tagging HTTP/1.1" 404 248 +botocore.parsers: DEBUG: Response headers: {'Content-Length': '248', 'x-amz-request-id': 'tx00000ebc589e4bcad8d86-0063354035-1157-default', 'Accept-Ranges': 'bytes', 'Content-Type': 'application/xml', 'Date': 'Thu, 29 Sep 2022 06:50:29 GMT', 'Connection': 'Keep-Alive'} +botocore.parsers: DEBUG: Response body: +b'NoSuchTagSetErrortest-client.0-2txq2dyjghs0vdf-335tx00000ebc589e4bcad8d86-0063354035-1157-default1157-default-default' +botocore.hooks: DEBUG: Event needs-retry.s3.GetBucketTagging: calling handler +botocore.retryhandler: DEBUG: No retry needed. +botocore.hooks: DEBUG: Event needs-retry.s3.GetBucketTagging: calling handler > +--------------------- >> end captured logging << ---------------------]]> + \ No newline at end of file diff --git a/teuthology/util/scanner.py b/teuthology/util/scanner.py new file mode 100644 index 0000000000..1e140d618c --- /dev/null +++ b/teuthology/util/scanner.py @@ -0,0 +1,149 @@ +import logging +import yaml +from typing import Optional, Tuple +from collections import defaultdict +from lxml import etree + +log = logging.getLogger(__name__) + + +class Scanner(): + def __init__(self, remote=None) -> None: + self.summary_data = [] + self.remote = remote + + def _parse(self, file_content) -> Tuple[str, dict]: + """ + This parses file_content and returns: + :returns: a message string + :returns: data dictionary with additional info + + Just an abstract method in Scanner class, + to be defined in inherited classes. + """ + raise NotImplementedError + + def scan_file(self, path: str) -> Optional[str]: + if not path: + return None + try: + file = self.remote._sftp_open_file(path, 'r') + file_content = file.read() + txt, data = self._parse(file_content) + if data: + data["file_path"] = path + self.summary_data += [data] + file.close() + return txt + except Exception as exc: + log.error(str(exc)) + + def scan_all_files(self, path_regex: str) -> [str]: + """ + Scans all files matching path_regex + and collect additional data in self.summary_data + + :param path_regex: Regex string to find all the files which have to be scanned. + Example: /path/to/dir/*.xml + """ + (_, stdout, _) = self.remote.ssh.exec_command(f'ls -d {path_regex}', timeout=200) + + files = stdout.read().decode().split('\n') + + extracted_txts = [] + for fpath in files: + txt = self.scan_file(fpath) + if txt: + extracted_txts += [txt] + return extracted_txts + + def write_summary(self, yaml_path: str) -> None: + """ + Create yaml file locally + with self.summary_data. + """ + if self.summary_data and yaml_path: + with open(yaml_path, 'a') as f: + yaml.safe_dump(self.summary_data, f, default_flow_style=False) + else: + log.info("summary_data or yaml_file is empty!") + + +class UnitTestScanner(Scanner): + def __init__(self, remote=None) -> None: + super().__init__(remote) + + def _parse(self, file_content: str) -> Tuple[Optional[str], Optional[dict]]: + xml_tree = etree.fromstring(file_content) + + failed_testcases = xml_tree.xpath('.//failure/.. | .//error/..') + if len(failed_testcases) == 0: + return None, None + + exception_txt = "" + error_data = defaultdict(list) + for testcase in failed_testcases: + testcase_name = testcase.get("name", "test-name") + testcase_suitename = testcase.get("classname", "suite-name") + for child in testcase: + if child.tag in ['failure', 'error']: + fault_kind = child.tag + reason = child.get('message', 'No message found in xml output, check logs.') + short_reason = reason[:200] + error_data[testcase_suitename] += [{ + "kind": fault_kind, + "testcase": testcase_name, + "message": reason, + }] + if not exception_txt: + exception_txt = f'{fault_kind.upper()}: Test `{testcase_name}` of `{testcase_suitename}`. Reason: {short_reason}.' + + return exception_txt, { "failed_testsuites": dict(error_data), "num_of_failures": len(failed_testcases) } + + def scan_and_write(self, path_regex: str, summary_path: str) -> Optional[str]: + """ + Scan all files matching 'path_regex' + and write summary in 'summary_path'. + """ + try: + errors = self.scan_all_files(path_regex) + self.write_summary(summary_path) + if errors: + return errors[0] + except Exception as scanner_exc: + log.error(str(scanner_exc)) + + +class ValgrindScanner(Scanner): + def __init__(self, remote=None) -> None: + super().__init__(remote) + + def _parse(self, file_content: str) -> Tuple[Optional[str], Optional[dict]]: + xml_tree = etree.fromstring(file_content) + if not xml_tree: + return None, None + + error_tree = xml_tree.find('error') + if error_tree is None: + return None, None + + error_data = { + "kind": error_tree.findtext("kind"), + "traceback": [], + } + for frame in error_tree.xpath("stack/frame"): + if len(error_data["traceback"]) >= 5: + break + curr_frame = { + "file": f"{frame.findtext('dir', '')}/{frame.findtext('file', '')}", + "line": frame.findtext("line", ''), + "function": frame.findtext("fn", ''), + } + error_data["traceback"].append(curr_frame) + + traceback_functions = "\n".join( + frame.get("function", "N/A") + for frame in error_data["traceback"][:3] + ) + exception_text = f"valgrind error: {error_data['kind']}\n{traceback_functions}" + return exception_text, error_data