X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=resources%2Ftools%2Fpresentation%2Finput_data_parser.py;h=bbbf0a9ae09a241c24ffd4d9ccec301b48122fa0;hb=c093f3f8a6c116c1897bb5f6793e99111ddb0682;hp=835609178c328f42801ea66d6e15fe6289501a5f;hpb=cafb31a9cb66cd4cbd9fe28bfd0ab2b83fec1e5d;p=csit.git diff --git a/resources/tools/presentation/input_data_parser.py b/resources/tools/presentation/input_data_parser.py index 835609178c..bbbf0a9ae0 100644 --- a/resources/tools/presentation/input_data_parser.py +++ b/resources/tools/presentation/input_data_parser.py @@ -19,11 +19,11 @@ - filter the data using tags, """ -import multiprocessing -import os import re +import resource import pandas as pd import logging +import prettytable from robot.api import ExecutionResult, ResultVisitor from robot import errors @@ -37,7 +37,6 @@ from json import loads from jumpavg.AvgStdevMetadataFactory import AvgStdevMetadataFactory from input_data_files import download_and_unzip_data_file -from utils import Worker # Separator used in file names @@ -282,8 +281,7 @@ class ExecutionChecker(ResultVisitor): REGEX_VERSION_VPP = re.compile(r"(return STDOUT Version:\s*|" r"VPP Version:\s*|VPP version:\s*)(.*)") - REGEX_VERSION_DPDK = re.compile(r"(return STDOUT testpmd)([\d\D\n]*)" - r"(RTE Version: 'DPDK )(.*)(')") + REGEX_VERSION_DPDK = re.compile(r"(DPDK version:\s*|DPDK Version:\s*)(.*)") REGEX_TCP = re.compile(r'Total\s(rps|cps|throughput):\s([0-9]*).*$') @@ -389,14 +387,12 @@ class ExecutionChecker(ResultVisitor): :returns: Nothing. """ - if msg.message.count("Arguments:"): - message = str(msg.message).replace(' ', '').replace('\n', '').\ - replace("'", '"').replace('b"', '"').\ - replace("honeycom", "honeycomb") - message = loads(message[11:-1]) + if msg.message.count("Setup of TG node"): + reg_tg_ip = re.compile( + r'Setup of TG node (\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}) done') try: - self._testbed = message["TG"]["host"] - except (KeyError, ValueError): + self._testbed = str(re.search(reg_tg_ip, msg.message).group(1)) + except (KeyError, ValueError, IndexError, AttributeError): pass finally: self._data["metadata"]["testbed"] = self._testbed @@ -426,10 +422,10 @@ class ExecutionChecker(ResultVisitor): :returns: Nothing. """ - if msg.message.count("return STDOUT testpmd"): + if msg.message.count("DPDK Version:"): try: self._version = str(re.search( - self.REGEX_VERSION_DPDK, msg.message). group(4)) + self.REGEX_VERSION_DPDK, msg.message). group(2)) self._data["metadata"]["version"] = self._version except IndexError: pass @@ -500,17 +496,54 @@ class ExecutionChecker(ResultVisitor): :type msg: Message :returns: Nothing. """ - if msg.message.count("Thread 0 vpp_main"): + if msg.message.count("Runtime:"): self._show_run_lookup_nr += 1 if self._lookup_kw_nr == 1 and self._show_run_lookup_nr == 1: self._data["tests"][self._test_ID]["show-run"] = str() if self._lookup_kw_nr > 1: self._msg_type = None if self._show_run_lookup_nr == 1: - text = msg.message.replace("vat# ", "").\ - replace("return STDOUT ", "").replace("\n\n", "\n").\ - replace('\n', ' |br| ').\ - replace('\r', '').replace('"', "'") + message = str(msg.message).replace(' ', '').replace('\n', '').\ + replace("'", '"').replace('b"', '"').replace('u"', '"')[8:] + runtime = loads(message) + try: + threads_nr = len(runtime[0]["clocks"]) + except (IndexError, KeyError): + return + tbl_hdr = ["Name", "Calls", "Vectors", "Suspends", "Clocks"] + table = [[tbl_hdr, ] for _ in range(threads_nr)] + for item in runtime: + for idx in range(threads_nr): + table[idx].append([ + item["name"], + item["calls"][idx], + item["vectors"][idx], + item["suspends"][idx], + item["clocks"][idx] + ]) + text = "" + for idx in range(threads_nr): + text += "Thread {idx} ".format(idx=idx) + text += "vpp_main\n" if idx == 0 else \ + "vpp_wk_{idx}\n".format(idx=idx-1) + txt_table = None + for row in table[idx]: + if txt_table is None: + txt_table = prettytable.PrettyTable(row) + else: + if any(row[1:]): + txt_table.add_row(row) + txt_table.set_style(prettytable.MSWORD_FRIENDLY) + txt_table.align["Name"] = "l" + txt_table.align["Calls"] = "r" + txt_table.align["Vectors"] = "r" + txt_table.align["Suspends"] = "r" + txt_table.align["Clocks"] = "r" + + text += txt_table.get_string(sortby="Name") + '\n' + + text = text.replace('\n', ' |br| ').replace('\r', '').\ + replace('"', "'") try: self._data["tests"][self._test_ID]["show-run"] += " |br| " self._data["tests"][self._test_ID]["show-run"] += \ @@ -948,7 +981,7 @@ class ExecutionChecker(ResultVisitor): self._lookup_kw_nr += 1 self._show_run_lookup_nr = 0 self._msg_type = "test-show-runtime" - elif test_kw.name.count("Start The L2fwd Test") and not self._version: + elif test_kw.name.count("Install Dpdk Test") and not self._version: self._msg_type = "dpdk-version" else: return @@ -1307,6 +1340,7 @@ class InputData(object): for build in builds: result = self._download_and_parse_build(job, build, repeat) + build_nr = result["build"]["build"] if result["data"]: data = result["data"] @@ -1321,12 +1355,15 @@ class InputData(object): if self._input_data.get(job, None) is None: self._input_data[job] = pd.Series() - self._input_data[job][str(build)] = build_data + self._input_data[job][str(build_nr)] = build_data self._cfg.set_input_file_name( - job, build, result["build"]["file-name"]) + job, build_nr, result["build"]["file-name"]) + + self._cfg.set_input_state(job, build_nr, result["state"]) - self._cfg.set_input_state(job, build, result["state"]) + logging.info("Memory allocation: {0:,d}MB".format( + resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000)) logging.info("Done.")