X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=resources%2Ftools%2Fpresentation%2Fgenerator_tables.py;h=c9205642c61c3dd8720bdf5752eab38c45d76d9f;hb=bf31370e5e5fa3b597229a631ca1d58788aae794;hp=0c52d204d78e3b31c08143d46327c5252ad9eb46;hpb=174ad309b359e9b323b97cae0a6877dce33deb5f;p=csit.git diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py index 0c52d204d7..c9205642c6 100644 --- a/resources/tools/presentation/generator_tables.py +++ b/resources/tools/presentation/generator_tables.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019 Cisco and/or its affiliates. +# Copyright (c) 2020 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -23,18 +23,20 @@ from collections import OrderedDict from xml.etree import ElementTree as ET from datetime import datetime as dt from datetime import timedelta +from copy import deepcopy import plotly.graph_objects as go import plotly.offline as ploff import pandas as pd from numpy import nan, isnan +from yaml import load, FullLoader, YAMLError -from pal_utils import mean, stdev, relative_change, classify_anomalies, \ +from pal_utils import mean, stdev, classify_anomalies, \ convert_csv_to_pretty_txt, relative_change_stdev -REGEX_NIC = re.compile(r'\d*ge\dp\d\D*\d*') +REGEX_NIC = re.compile(r'(\d*ge\dp\d\D*\d*[a-z]*)') def generate_tables(spec, data): @@ -47,7 +49,6 @@ def generate_tables(spec, data): """ generator = { - u"table_details": table_details, u"table_merged_details": table_merged_details, u"table_perf_comparison": table_perf_comparison, u"table_perf_comparison_nic": table_perf_comparison_nic, @@ -57,7 +58,9 @@ def generate_tables(spec, data): u"table_perf_trending_dash_html": table_perf_trending_dash_html, u"table_last_failed_tests": table_last_failed_tests, u"table_failed_tests": table_failed_tests, - u"table_failed_tests_html": table_failed_tests_html + u"table_failed_tests_html": table_failed_tests_html, + u"table_oper_data_html": table_oper_data_html, + u"table_comparison": table_comparison } logging.info(u"Generating the tables ...") @@ -72,8 +75,8 @@ def generate_tables(spec, data): logging.info(u"Done.") -def table_details(table, input_data): - """Generate the table(s) with algorithm: table_detailed_test_results +def table_oper_data_html(table, input_data): + """Generate the table(s) with algorithm: html_table_oper_data specified in the specification file. :param table: Table to generate. @@ -83,66 +86,196 @@ def table_details(table, input_data): """ logging.info(f" Generating the table {table.get(u'title', u'')} ...") - # Transform the data logging.info( f" Creating the data set for the {table.get(u'type', u'')} " f"{table.get(u'title', u'')}." ) - data = input_data.filter_data(table) + data = input_data.filter_data( + table, + params=[u"name", u"parent", u"show-run", u"type"], + continue_on_error=True + ) + if data.empty: + return + data = input_data.merge_data(data) - # Prepare the header of the tables - header = list() - for column in table[u"columns"]: - header.append( - u'"{0}"'.format(str(column[u"title"]).replace(u'"', u'""')) + sort_tests = table.get(u"sort", None) + if sort_tests: + args = dict( + inplace=True, + ascending=(sort_tests == u"ascending") ) + data.sort_index(**args) - # Generate the data for the table according to the model in the table - # specification - job = list(table[u"data"].keys())[0] - build = str(table[u"data"][job][0]) - try: - suites = input_data.suites(job, build) - except KeyError: - logging.error( - u" No data available. The table will not be generated." - ) + suites = input_data.filter_data( + table, + continue_on_error=True, + data_set=u"suites" + ) + if suites.empty: return + suites = input_data.merge_data(suites) - for suite in suites.values: - # Generate data - suite_name = suite[u"name"] - table_lst = list() - for test in data[job][build].keys(): - if data[job][build][test][u"parent"] not in suite_name: + def _generate_html_table(tst_data): + """Generate an HTML table with operational data for the given test. + + :param tst_data: Test data to be used to generate the table. + :type tst_data: pandas.Series + :returns: HTML table with operational data. + :rtype: str + """ + + colors = { + u"header": u"#7eade7", + u"empty": u"#ffffff", + u"body": (u"#e9f1fb", u"#d4e4f7") + } + + tbl = ET.Element(u"table", attrib=dict(width=u"100%", border=u"0")) + + trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])) + thead = ET.SubElement( + trow, u"th", attrib=dict(align=u"left", colspan=u"6") + ) + thead.text = tst_data[u"name"] + + trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])) + thead = ET.SubElement( + trow, u"th", attrib=dict(align=u"left", colspan=u"6") + ) + thead.text = u"\t" + + if tst_data.get(u"show-run", u"No Data") == u"No Data": + trow = ET.SubElement( + tbl, u"tr", attrib=dict(bgcolor=colors[u"header"]) + ) + tcol = ET.SubElement( + trow, u"td", attrib=dict(align=u"left", colspan=u"6") + ) + tcol.text = u"No Data" + + trow = ET.SubElement( + tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]) + ) + thead = ET.SubElement( + trow, u"th", attrib=dict(align=u"left", colspan=u"6") + ) + font = ET.SubElement( + thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff") + ) + font.text = u"." + return str(ET.tostring(tbl, encoding=u"unicode")) + + tbl_hdr = ( + u"Name", + u"Nr of Vectors", + u"Nr of Packets", + u"Suspends", + u"Cycles per Packet", + u"Average Vector Size" + ) + + for dut_data in tst_data[u"show-run"].values(): + trow = ET.SubElement( + tbl, u"tr", attrib=dict(bgcolor=colors[u"header"]) + ) + tcol = ET.SubElement( + trow, u"td", attrib=dict(align=u"left", colspan=u"6") + ) + if dut_data.get(u"threads", None) is None: + tcol.text = u"No Data" continue - row_lst = list() - for column in table[u"columns"]: - try: - col_data = str(data[job][build][test][column[ - u"data"].split(" ")[1]]).replace(u'"', u'""') - if column[u"data"].split(u" ")[1] in \ - (u"conf-history", u"show-run"): - col_data = col_data.replace(u" |br| ", u"", ) - col_data = f" |prein| {col_data[:-5]} |preout| " - row_lst.append(f'"{col_data}"') - except KeyError: - row_lst.append(u"No data") - table_lst.append(row_lst) - # Write the data to file - if table_lst: - file_name = ( - f"{table[u'output-file']}_{suite_name}" - f"{table[u'output-file-ext']}" + bold = ET.SubElement(tcol, u"b") + bold.text = ( + f"Host IP: {dut_data.get(u'host', '')}, " + f"Socket: {dut_data.get(u'socket', '')}" ) - logging.info(f" Writing file: {file_name}") - with open(file_name, u"w") as file_handler: - file_handler.write(u",".join(header) + u"\n") - for item in table_lst: - file_handler.write(u",".join(item) + u"\n") + trow = ET.SubElement( + tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]) + ) + thead = ET.SubElement( + trow, u"th", attrib=dict(align=u"left", colspan=u"6") + ) + thead.text = u"\t" + + for thread_nr, thread in dut_data[u"threads"].items(): + trow = ET.SubElement( + tbl, u"tr", attrib=dict(bgcolor=colors[u"header"]) + ) + tcol = ET.SubElement( + trow, u"td", attrib=dict(align=u"left", colspan=u"6") + ) + bold = ET.SubElement(tcol, u"b") + bold.text = u"main" if thread_nr == 0 else f"worker_{thread_nr}" + trow = ET.SubElement( + tbl, u"tr", attrib=dict(bgcolor=colors[u"header"]) + ) + for idx, col in enumerate(tbl_hdr): + tcol = ET.SubElement( + trow, u"td", + attrib=dict(align=u"right" if idx else u"left") + ) + font = ET.SubElement( + tcol, u"font", attrib=dict(size=u"2") + ) + bold = ET.SubElement(font, u"b") + bold.text = col + for row_nr, row in enumerate(thread): + trow = ET.SubElement( + tbl, u"tr", + attrib=dict(bgcolor=colors[u"body"][row_nr % 2]) + ) + for idx, col in enumerate(row): + tcol = ET.SubElement( + trow, u"td", + attrib=dict(align=u"right" if idx else u"left") + ) + font = ET.SubElement( + tcol, u"font", attrib=dict(size=u"2") + ) + if isinstance(col, float): + font.text = f"{col:.2f}" + else: + font.text = str(col) + trow = ET.SubElement( + tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]) + ) + thead = ET.SubElement( + trow, u"th", attrib=dict(align=u"left", colspan=u"6") + ) + thead.text = u"\t" + + trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])) + thead = ET.SubElement( + trow, u"th", attrib=dict(align=u"left", colspan=u"6") + ) + font = ET.SubElement( + thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff") + ) + font.text = u"." + return str(ET.tostring(tbl, encoding=u"unicode")) + + for suite in suites.values: + html_table = str() + for test_data in data.values: + if test_data[u"parent"] not in suite[u"name"]: + continue + html_table += _generate_html_table(test_data) + if not html_table: + continue + try: + file_name = f"{table[u'output-file']}{suite[u'name']}.rst" + with open(f"{file_name}", u'w') as html_file: + logging.info(f" Writing file: {file_name}") + html_file.write(u".. raw:: html\n\n\t") + html_file.write(html_table) + html_file.write(u"\n\t
\n\n\n" + u".. |preout| raw:: html\n\n\n\n" + ) + rst_file.write( + u".. raw:: html\n\n" + f' \n\n' + ) + if legend: + rst_file.write(legend[1:].replace(u"\n", u" |br| ")) + if footnote: + rst_file.write(footnote.replace(u"\n", u" |br| ")[1:]) def table_perf_comparison(table, input_data): @@ -438,55 +662,108 @@ def table_perf_comparison(table, input_data): # Prepare the header of the tables try: - header = [u"Test case", ] + header = [u"Test Case", ] + legend = u"\nLegend:\n" - if table[u"include-tests"] == u"MRR": - hdr_param = u"Rec Rate" - else: - hdr_param = u"Thput" + rca_data = None + rca = table.get(u"rca", None) + if rca: + try: + with open(rca.get(u"data-file", u""), u"r") as rca_file: + rca_data = load(rca_file, Loader=FullLoader) + header.insert(0, rca.get(u"title", u"RCA")) + legend += ( + u"RCA: Reference to the Root Cause Analysis, see below.\n" + ) + except (YAMLError, IOError) as err: + logging.warning(repr(err)) history = table.get(u"history", list()) for item in history: header.extend( [ - f"{item[u'title']} {hdr_param} [Mpps]", - f"{item[u'title']} Stdev [Mpps]" + f"{item[u'title']} Avg({table[u'include-tests']})", + f"{item[u'title']} Stdev({table[u'include-tests']})" ] ) + legend += ( + f"{item[u'title']} Avg({table[u'include-tests']}): " + f"Mean value of {table[u'include-tests']} [Mpps] computed from " + f"a series of runs of the listed tests executed against " + f"{item[u'title']}.\n" + f"{item[u'title']} Stdev({table[u'include-tests']}): " + f"Standard deviation value of {table[u'include-tests']} [Mpps] " + f"computed from a series of runs of the listed tests executed " + f"against {item[u'title']}.\n" + ) header.extend( [ - f"{table[u'reference'][u'title']} {hdr_param} [Mpps]", - f"{table[u'reference'][u'title']} Stdev [Mpps]", - f"{table[u'compare'][u'title']} {hdr_param} [Mpps]", - f"{table[u'compare'][u'title']} Stdev [Mpps]", - u"Delta [%]" + f"{table[u'reference'][u'title']} " + f"Avg({table[u'include-tests']})", + f"{table[u'reference'][u'title']} " + f"Stdev({table[u'include-tests']})", + f"{table[u'compare'][u'title']} " + f"Avg({table[u'include-tests']})", + f"{table[u'compare'][u'title']} " + f"Stdev({table[u'include-tests']})", + f"Diff({table[u'reference'][u'title']}," + f"{table[u'compare'][u'title']})", + u"Stdev(Diff)" ] ) - header_str = u",".join(header) + u"\n" + header_str = u";".join(header) + u"\n" + legend += ( + f"{table[u'reference'][u'title']} " + f"Avg({table[u'include-tests']}): " + f"Mean value of {table[u'include-tests']} [Mpps] computed from a " + f"series of runs of the listed tests executed against " + f"{table[u'reference'][u'title']}.\n" + f"{table[u'reference'][u'title']} " + f"Stdev({table[u'include-tests']}): " + f"Standard deviation value of {table[u'include-tests']} [Mpps] " + f"computed from a series of runs of the listed tests executed " + f"against {table[u'reference'][u'title']}.\n" + f"{table[u'compare'][u'title']} " + f"Avg({table[u'include-tests']}): " + f"Mean value of {table[u'include-tests']} [Mpps] computed from a " + f"series of runs of the listed tests executed against " + f"{table[u'compare'][u'title']}.\n" + f"{table[u'compare'][u'title']} " + f"Stdev({table[u'include-tests']}): " + f"Standard deviation value of {table[u'include-tests']} [Mpps] " + f"computed from a series of runs of the listed tests executed " + f"against {table[u'compare'][u'title']}.\n" + f"Diff({table[u'reference'][u'title']}," + f"{table[u'compare'][u'title']}): " + f"Percentage change calculated for mean values.\n" + u"Stdev(Diff): " + u"Standard deviation of percentage change calculated for mean " + u"values.\n" + u"NT: Not Tested\n" + ) except (AttributeError, KeyError) as err: logging.error(f"The model is invalid, missing parameter: {repr(err)}") return # Prepare data to the table: tbl_dict = dict() - topo = "" for job, builds in table[u"reference"][u"data"].items(): - topo = u"2n-skx" if u"2n-skx" in job else u"" for build in builds: for tst_name, tst_data in data[job][str(build)].items(): tst_name_mod = _tpc_modify_test_name(tst_name) - if u"across topologies" in table[u"title"].lower(): + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: - groups = re.search(REGEX_NIC, tst_data[u"parent"]) - nic = groups.group(0) if groups else u"" - name = \ - f"{nic}-{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}" + name = tst_data[u'name'].rsplit(u'-', 1)[0] if u"across testbeds" in table[u"title"].lower() or \ u"across topologies" in table[u"title"].lower(): name = _tpc_modify_displayed_test_name(name) tbl_dict[tst_name_mod] = { u"name": name, + u"replace-ref": True, + u"replace-cmp": True, u"ref-data": list(), u"cmp-data": list() } @@ -494,22 +771,57 @@ def table_perf_comparison(table, input_data): src=tst_data, include_tests=table[u"include-tests"]) + replacement = table[u"reference"].get(u"data-replacement", None) + if replacement: + rpl_data = input_data.filter_data( + table, data=replacement, continue_on_error=True) + for job, builds in replacement.items(): + for build in builds: + for tst_name, tst_data in rpl_data[job][str(build)].items(): + tst_name_mod = _tpc_modify_test_name(tst_name) + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): + tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") + if tbl_dict.get(tst_name_mod, None) is None: + name = tst_data[u'name'].rsplit(u'-', 1)[0] + if u"across testbeds" in table[u"title"].lower() or \ + u"across topologies" in table[u"title"].lower(): + name = _tpc_modify_displayed_test_name(name) + tbl_dict[tst_name_mod] = { + u"name": name, + u"replace-ref": False, + u"replace-cmp": True, + u"ref-data": list(), + u"cmp-data": list() + } + if tbl_dict[tst_name_mod][u"replace-ref"]: + tbl_dict[tst_name_mod][u"replace-ref"] = False + tbl_dict[tst_name_mod][u"ref-data"] = list() + + _tpc_insert_data( + target=tbl_dict[tst_name_mod][u"ref-data"], + src=tst_data, + include_tests=table[u"include-tests"] + ) + for job, builds in table[u"compare"][u"data"].items(): for build in builds: for tst_name, tst_data in data[job][str(build)].items(): tst_name_mod = _tpc_modify_test_name(tst_name) - if u"across topologies" in table[u"title"].lower(): + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: - groups = re.search(REGEX_NIC, tst_data[u"parent"]) - nic = groups.group(0) if groups else u"" - name = \ - f"{nic}-{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}" + name = tst_data[u'name'].rsplit(u'-', 1)[0] if u"across testbeds" in table[u"title"].lower() or \ u"across topologies" in table[u"title"].lower(): name = _tpc_modify_displayed_test_name(name) tbl_dict[tst_name_mod] = { u"name": name, + u"replace-ref": False, + u"replace-cmp": True, u"ref-data": list(), u"cmp-data": list() } @@ -521,28 +833,30 @@ def table_perf_comparison(table, input_data): replacement = table[u"compare"].get(u"data-replacement", None) if replacement: - create_new_list = True rpl_data = input_data.filter_data( table, data=replacement, continue_on_error=True) for job, builds in replacement.items(): for build in builds: for tst_name, tst_data in rpl_data[job][str(build)].items(): tst_name_mod = _tpc_modify_test_name(tst_name) - if u"across topologies" in table[u"title"].lower(): + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: - name = \ - f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}" + name = tst_data[u'name'].rsplit(u'-', 1)[0] if u"across testbeds" in table[u"title"].lower() or \ u"across topologies" in table[u"title"].lower(): name = _tpc_modify_displayed_test_name(name) tbl_dict[tst_name_mod] = { u"name": name, + u"replace-ref": False, + u"replace-cmp": False, u"ref-data": list(), u"cmp-data": list() } - if create_new_list: - create_new_list = False + if tbl_dict[tst_name_mod][u"replace-cmp"]: + tbl_dict[tst_name_mod][u"replace-cmp"] = False tbl_dict[tst_name_mod][u"cmp-data"] = list() _tpc_insert_data( @@ -556,7 +870,9 @@ def table_perf_comparison(table, input_data): for build in builds: for tst_name, tst_data in data[job][str(build)].items(): tst_name_mod = _tpc_modify_test_name(tst_name) - if u"across topologies" in table[u"title"].lower(): + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: continue @@ -568,7 +884,8 @@ def table_perf_comparison(table, input_data): u"title"]] = list() try: if table[u"include-tests"] == u"MRR": - res = tst_data[u"result"][u"receive-rate"] + res = (tst_data[u"result"][u"receive-rate"], + tst_data[u"result"][u"receive-stdev"]) elif table[u"include-tests"] == u"PDR": res = tst_data[u"throughput"][u"PDR"][u"LOWER"] elif table[u"include-tests"] == u"NDR": @@ -581,71 +898,102 @@ def table_perf_comparison(table, input_data): pass tbl_lst = list() - footnote = False for tst_name in tbl_dict: item = [tbl_dict[tst_name][u"name"], ] if history: if tbl_dict[tst_name].get(u"history", None) is not None: for hist_data in tbl_dict[tst_name][u"history"].values(): if hist_data: - item.append(round(mean(hist_data) / 1000000, 2)) - item.append(round(stdev(hist_data) / 1000000, 2)) + if table[u"include-tests"] == u"MRR": + item.append(round(hist_data[0][0] / 1e6, 1)) + item.append(round(hist_data[0][1] / 1e6, 1)) + else: + item.append(round(mean(hist_data) / 1e6, 1)) + item.append(round(stdev(hist_data) / 1e6, 1)) else: - item.extend([u"Not tested", u"Not tested"]) + item.extend([u"NT", u"NT"]) else: - item.extend([u"Not tested", u"Not tested"]) - data_t = tbl_dict[tst_name][u"ref-data"] - if data_t: - item.append(round(mean(data_t) / 1000000, 2)) - item.append(round(stdev(data_t) / 1000000, 2)) + item.extend([u"NT", u"NT"]) + data_r = tbl_dict[tst_name][u"ref-data"] + if data_r: + if table[u"include-tests"] == u"MRR": + data_r_mean = data_r[0][0] + data_r_stdev = data_r[0][1] + else: + data_r_mean = mean(data_r) + data_r_stdev = stdev(data_r) + item.append(round(data_r_mean / 1e6, 1)) + item.append(round(data_r_stdev / 1e6, 1)) else: - item.extend([u"Not tested", u"Not tested"]) - data_t = tbl_dict[tst_name][u"cmp-data"] - if data_t: - item.append(round(mean(data_t) / 1000000, 2)) - item.append(round(stdev(data_t) / 1000000, 2)) + data_r_mean = None + data_r_stdev = None + item.extend([u"NT", u"NT"]) + data_c = tbl_dict[tst_name][u"cmp-data"] + if data_c: + if table[u"include-tests"] == u"MRR": + data_c_mean = data_c[0][0] + data_c_stdev = data_c[0][1] + else: + data_c_mean = mean(data_c) + data_c_stdev = stdev(data_c) + item.append(round(data_c_mean / 1e6, 1)) + item.append(round(data_c_stdev / 1e6, 1)) else: - item.extend([u"Not tested", u"Not tested"]) - if item[-2] == u"Not tested": + data_c_mean = None + data_c_stdev = None + item.extend([u"NT", u"NT"]) + if item[-2] == u"NT": pass - elif item[-4] == u"Not tested": - item.append(u"New in CSIT-1908") - elif topo == u"2n-skx" and u"dot1q" in tbl_dict[tst_name][u"name"]: - item.append(u"See footnote [1]") - footnote = True - elif item[-4] != 0: - item.append(int(relative_change(float(item[-4]), float(item[-2])))) - if (len(item) == len(header)) and (item[-3] != u"Not tested"): + elif item[-4] == u"NT": + item.append(u"New in CSIT-2001") + item.append(u"New in CSIT-2001") + elif data_r_mean is not None and data_c_mean is not None: + delta, d_stdev = relative_change_stdev( + data_r_mean, data_c_mean, data_r_stdev, data_c_stdev + ) + try: + item.append(round(delta)) + except ValueError: + item.append(delta) + try: + item.append(round(d_stdev)) + except ValueError: + item.append(d_stdev) + if rca_data: + rca_nr = rca_data.get(item[0], u"-") + item.insert(0, f"[{rca_nr}]" if rca_nr != u"-" else u"-") + if (len(item) == len(header)) and (item[-4] != u"NT"): tbl_lst.append(item) tbl_lst = _tpc_sort_table(tbl_lst) # Generate csv tables: csv_file = f"{table[u'output-file']}.csv" - with open(csv_file, u"w") as file_handler: + with open(csv_file, u"wt") as file_handler: file_handler.write(header_str) for test in tbl_lst: - file_handler.write(u",".join([str(item) for item in test]) + u"\n") + file_handler.write(u";".join([str(item) for item in test]) + u"\n") txt_file_name = f"{table[u'output-file']}.txt" - convert_csv_to_pretty_txt(csv_file, txt_file_name) - - if footnote: - with open(txt_file_name, u'a') as txt_file: - txt_file.writelines([ - u"\nFootnotes:\n", - u"[1] CSIT-1908 changed test methodology of dot1q tests in " - u"2-node testbeds, dot1q encapsulation is now used on both " - u"links of SUT.\n", - u" Previously dot1q was used only on a single link with the " - u"other link carrying untagged Ethernet frames. This changes " - u"results\n", - u" in slightly lower throughput in CSIT-1908 for these " - u"tests. See release notes." - ]) + convert_csv_to_pretty_txt(csv_file, txt_file_name, delimiter=u";") + + footnote = u"" + with open(txt_file_name, u'a') as txt_file: + txt_file.write(legend) + if rca_data: + footnote = rca_data.get(u"footnote", u"") + if footnote: + txt_file.write(footnote) + txt_file.write(u":END") # Generate html table: - _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html") + _tpc_generate_html_table( + header, + tbl_lst, + table[u'output-file'], + legend=legend, + footnote=footnote + ) def table_perf_comparison_nic(table, input_data): @@ -669,54 +1017,110 @@ def table_perf_comparison_nic(table, input_data): # Prepare the header of the tables try: - header = [u"Test case", ] + header = [u"Test Case", ] + legend = u"\nLegend:\n" - if table[u"include-tests"] == u"MRR": - hdr_param = u"Rec Rate" - else: - hdr_param = u"Thput" + rca_data = None + rca = table.get(u"rca", None) + if rca: + try: + with open(rca.get(u"data-file", ""), u"r") as rca_file: + rca_data = load(rca_file, Loader=FullLoader) + header.insert(0, rca.get(u"title", "RCA")) + legend += ( + u"RCA: Reference to the Root Cause Analysis, see below.\n" + ) + except (YAMLError, IOError) as err: + logging.warning(repr(err)) history = table.get(u"history", list()) for item in history: header.extend( [ - f"{item[u'title']} {hdr_param} [Mpps]", - f"{item[u'title']} Stdev [Mpps]" + f"{item[u'title']} Avg({table[u'include-tests']})", + f"{item[u'title']} Stdev({table[u'include-tests']})" ] ) + legend += ( + f"{item[u'title']} Avg({table[u'include-tests']}): " + f"Mean value of {table[u'include-tests']} [Mpps] computed from " + f"a series of runs of the listed tests executed against " + f"{item[u'title']}.\n" + f"{item[u'title']} Stdev({table[u'include-tests']}): " + f"Standard deviation value of {table[u'include-tests']} [Mpps] " + f"computed from a series of runs of the listed tests executed " + f"against {item[u'title']}.\n" + ) header.extend( [ - f"{table[u'reference'][u'title']} {hdr_param} [Mpps]", - f"{table[u'reference'][u'title']} Stdev [Mpps]", - f"{table[u'compare'][u'title']} {hdr_param} [Mpps]", - f"{table[u'compare'][u'title']} Stdev [Mpps]", - u"Delta [%]" + f"{table[u'reference'][u'title']} " + f"Avg({table[u'include-tests']})", + f"{table[u'reference'][u'title']} " + f"Stdev({table[u'include-tests']})", + f"{table[u'compare'][u'title']} " + f"Avg({table[u'include-tests']})", + f"{table[u'compare'][u'title']} " + f"Stdev({table[u'include-tests']})", + f"Diff({table[u'reference'][u'title']}," + f"{table[u'compare'][u'title']})", + u"Stdev(Diff)" ] ) - header_str = u",".join(header) + u"\n" + header_str = u";".join(header) + u"\n" + legend += ( + f"{table[u'reference'][u'title']} " + f"Avg({table[u'include-tests']}): " + f"Mean value of {table[u'include-tests']} [Mpps] computed from a " + f"series of runs of the listed tests executed against " + f"{table[u'reference'][u'title']}.\n" + f"{table[u'reference'][u'title']} " + f"Stdev({table[u'include-tests']}): " + f"Standard deviation value of {table[u'include-tests']} [Mpps] " + f"computed from a series of runs of the listed tests executed " + f"against {table[u'reference'][u'title']}.\n" + f"{table[u'compare'][u'title']} " + f"Avg({table[u'include-tests']}): " + f"Mean value of {table[u'include-tests']} [Mpps] computed from a " + f"series of runs of the listed tests executed against " + f"{table[u'compare'][u'title']}.\n" + f"{table[u'compare'][u'title']} " + f"Stdev({table[u'include-tests']}): " + f"Standard deviation value of {table[u'include-tests']} [Mpps] " + f"computed from a series of runs of the listed tests executed " + f"against {table[u'compare'][u'title']}.\n" + f"Diff({table[u'reference'][u'title']}," + f"{table[u'compare'][u'title']}): " + f"Percentage change calculated for mean values.\n" + u"Stdev(Diff): " + u"Standard deviation of percentage change calculated for mean " + u"values.\n" + u"NT: Not Tested\n" + ) except (AttributeError, KeyError) as err: logging.error(f"The model is invalid, missing parameter: {repr(err)}") return # Prepare data to the table: tbl_dict = dict() - topo = u"" for job, builds in table[u"reference"][u"data"].items(): - topo = u"2n-skx" if u"2n-skx" in job else u"" for build in builds: for tst_name, tst_data in data[job][str(build)].items(): if table[u"reference"][u"nic"] not in tst_data[u"tags"]: continue tst_name_mod = _tpc_modify_test_name(tst_name) - if u"across topologies" in table[u"title"].lower(): + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: - name = f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}" + name = tst_data[u'name'].rsplit(u'-', 1)[0] if u"across testbeds" in table[u"title"].lower() or \ u"across topologies" in table[u"title"].lower(): name = _tpc_modify_displayed_test_name(name) tbl_dict[tst_name_mod] = { u"name": name, + u"replace-ref": True, + u"replace-cmp": True, u"ref-data": list(), u"cmp-data": list() } @@ -726,21 +1130,61 @@ def table_perf_comparison_nic(table, input_data): include_tests=table[u"include-tests"] ) + replacement = table[u"reference"].get(u"data-replacement", None) + if replacement: + rpl_data = input_data.filter_data( + table, data=replacement, continue_on_error=True) + for job, builds in replacement.items(): + for build in builds: + for tst_name, tst_data in rpl_data[job][str(build)].items(): + if table[u"reference"][u"nic"] not in tst_data[u"tags"]: + continue + tst_name_mod = _tpc_modify_test_name(tst_name) + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): + tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") + if tbl_dict.get(tst_name_mod, None) is None: + name = tst_data[u'name'].rsplit(u'-', 1)[0] + if u"across testbeds" in table[u"title"].lower() or \ + u"across topologies" in table[u"title"].lower(): + name = _tpc_modify_displayed_test_name(name) + tbl_dict[tst_name_mod] = { + u"name": name, + u"replace-ref": False, + u"replace-cmp": True, + u"ref-data": list(), + u"cmp-data": list() + } + if tbl_dict[tst_name_mod][u"replace-ref"]: + tbl_dict[tst_name_mod][u"replace-ref"] = False + tbl_dict[tst_name_mod][u"ref-data"] = list() + + _tpc_insert_data( + target=tbl_dict[tst_name_mod][u"ref-data"], + src=tst_data, + include_tests=table[u"include-tests"] + ) + for job, builds in table[u"compare"][u"data"].items(): for build in builds: for tst_name, tst_data in data[job][str(build)].items(): if table[u"compare"][u"nic"] not in tst_data[u"tags"]: continue tst_name_mod = _tpc_modify_test_name(tst_name) - if u"across topologies" in table[u"title"].lower(): + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: - name = f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}" + name = tst_data[u'name'].rsplit(u'-', 1)[0] if u"across testbeds" in table[u"title"].lower() or \ u"across topologies" in table[u"title"].lower(): name = _tpc_modify_displayed_test_name(name) tbl_dict[tst_name_mod] = { u"name": name, + u"replace-ref": False, + u"replace-cmp": True, u"ref-data": list(), u"cmp-data": list() } @@ -752,7 +1196,6 @@ def table_perf_comparison_nic(table, input_data): replacement = table[u"compare"].get(u"data-replacement", None) if replacement: - create_new_list = True rpl_data = input_data.filter_data( table, data=replacement, continue_on_error=True) for job, builds in replacement.items(): @@ -761,21 +1204,24 @@ def table_perf_comparison_nic(table, input_data): if table[u"compare"][u"nic"] not in tst_data[u"tags"]: continue tst_name_mod = _tpc_modify_test_name(tst_name) - if u"across topologies" in table[u"title"].lower(): + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: - name = \ - f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}" + name = tst_data[u'name'].rsplit(u'-', 1)[0] if u"across testbeds" in table[u"title"].lower() or \ u"across topologies" in table[u"title"].lower(): name = _tpc_modify_displayed_test_name(name) tbl_dict[tst_name_mod] = { u"name": name, + u"replace-ref": False, + u"replace-cmp": False, u"ref-data": list(), u"cmp-data": list() } - if create_new_list: - create_new_list = False + if tbl_dict[tst_name_mod][u"replace-cmp"]: + tbl_dict[tst_name_mod][u"replace-cmp"] = False tbl_dict[tst_name_mod][u"cmp-data"] = list() _tpc_insert_data( @@ -791,7 +1237,9 @@ def table_perf_comparison_nic(table, input_data): if item[u"nic"] not in tst_data[u"tags"]: continue tst_name_mod = _tpc_modify_test_name(tst_name) - if u"across topologies" in table[u"title"].lower(): + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: continue @@ -803,7 +1251,8 @@ def table_perf_comparison_nic(table, input_data): u"title"]] = list() try: if table[u"include-tests"] == u"MRR": - res = tst_data[u"result"][u"receive-rate"] + res = (tst_data[u"result"][u"receive-rate"], + tst_data[u"result"][u"receive-stdev"]) elif table[u"include-tests"] == u"PDR": res = tst_data[u"throughput"][u"PDR"][u"LOWER"] elif table[u"include-tests"] == u"NDR": @@ -816,71 +1265,102 @@ def table_perf_comparison_nic(table, input_data): pass tbl_lst = list() - footnote = False for tst_name in tbl_dict: item = [tbl_dict[tst_name][u"name"], ] if history: if tbl_dict[tst_name].get(u"history", None) is not None: for hist_data in tbl_dict[tst_name][u"history"].values(): if hist_data: - item.append(round(mean(hist_data) / 1000000, 2)) - item.append(round(stdev(hist_data) / 1000000, 2)) + if table[u"include-tests"] == u"MRR": + item.append(round(hist_data[0][0] / 1e6, 1)) + item.append(round(hist_data[0][1] / 1e6, 1)) + else: + item.append(round(mean(hist_data) / 1e6, 1)) + item.append(round(stdev(hist_data) / 1e6, 1)) else: - item.extend([u"Not tested", u"Not tested"]) + item.extend([u"NT", u"NT"]) + else: + item.extend([u"NT", u"NT"]) + data_r = tbl_dict[tst_name][u"ref-data"] + if data_r: + if table[u"include-tests"] == u"MRR": + data_r_mean = data_r[0][0] + data_r_stdev = data_r[0][1] else: - item.extend([u"Not tested", u"Not tested"]) - data_t = tbl_dict[tst_name][u"ref-data"] - if data_t: - item.append(round(mean(data_t) / 1000000, 2)) - item.append(round(stdev(data_t) / 1000000, 2)) + data_r_mean = mean(data_r) + data_r_stdev = stdev(data_r) + item.append(round(data_r_mean / 1e6, 1)) + item.append(round(data_r_stdev / 1e6, 1)) else: - item.extend([u"Not tested", u"Not tested"]) - data_t = tbl_dict[tst_name][u"cmp-data"] - if data_t: - item.append(round(mean(data_t) / 1000000, 2)) - item.append(round(stdev(data_t) / 1000000, 2)) + data_r_mean = None + data_r_stdev = None + item.extend([u"NT", u"NT"]) + data_c = tbl_dict[tst_name][u"cmp-data"] + if data_c: + if table[u"include-tests"] == u"MRR": + data_c_mean = data_c[0][0] + data_c_stdev = data_c[0][1] + else: + data_c_mean = mean(data_c) + data_c_stdev = stdev(data_c) + item.append(round(data_c_mean / 1e6, 1)) + item.append(round(data_c_stdev / 1e6, 1)) else: - item.extend([u"Not tested", u"Not tested"]) - if item[-2] == u"Not tested": + data_c_mean = None + data_c_stdev = None + item.extend([u"NT", u"NT"]) + if item[-2] == u"NT": pass - elif item[-4] == u"Not tested": - item.append(u"New in CSIT-1908") - elif topo == u"2n-skx" and u"dot1q" in tbl_dict[tst_name][u"name"]: - item.append(u"See footnote [1]") - footnote = True - elif item[-4] != 0: - item.append(int(relative_change(float(item[-4]), float(item[-2])))) - if (len(item) == len(header)) and (item[-3] != u"Not tested"): + elif item[-4] == u"NT": + item.append(u"New in CSIT-2001") + item.append(u"New in CSIT-2001") + elif data_r_mean is not None and data_c_mean is not None: + delta, d_stdev = relative_change_stdev( + data_r_mean, data_c_mean, data_r_stdev, data_c_stdev + ) + try: + item.append(round(delta)) + except ValueError: + item.append(delta) + try: + item.append(round(d_stdev)) + except ValueError: + item.append(d_stdev) + if rca_data: + rca_nr = rca_data.get(item[0], u"-") + item.insert(0, f"[{rca_nr}]" if rca_nr != u"-" else u"-") + if (len(item) == len(header)) and (item[-4] != u"NT"): tbl_lst.append(item) tbl_lst = _tpc_sort_table(tbl_lst) # Generate csv tables: csv_file = f"{table[u'output-file']}.csv" - with open(csv_file, u"w") as file_handler: + with open(csv_file, u"wt") as file_handler: file_handler.write(header_str) for test in tbl_lst: - file_handler.write(u",".join([str(item) for item in test]) + u"\n") + file_handler.write(u";".join([str(item) for item in test]) + u"\n") txt_file_name = f"{table[u'output-file']}.txt" - convert_csv_to_pretty_txt(csv_file, txt_file_name) - - if footnote: - with open(txt_file_name, u'a') as txt_file: - txt_file.writelines([ - u"\nFootnotes:\n", - u"[1] CSIT-1908 changed test methodology of dot1q tests in " - u"2-node testbeds, dot1q encapsulation is now used on both " - u"links of SUT.\n", - u" Previously dot1q was used only on a single link with the " - u"other link carrying untagged Ethernet frames. This changes " - u"results\n", - u" in slightly lower throughput in CSIT-1908 for these " - u"tests. See release notes." - ]) + convert_csv_to_pretty_txt(csv_file, txt_file_name, delimiter=u";") + + footnote = u"" + with open(txt_file_name, u'a') as txt_file: + txt_file.write(legend) + if rca_data: + footnote = rca_data.get(u"footnote", u"") + if footnote: + txt_file.write(footnote) + txt_file.write(u":END") # Generate html table: - _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html") + _tpc_generate_html_table( + header, + tbl_lst, + table[u'output-file'], + legend=legend, + footnote=footnote + ) def table_nics_comparison(table, input_data): @@ -904,21 +1384,49 @@ def table_nics_comparison(table, input_data): # Prepare the header of the tables try: - header = [u"Test case", ] - - if table[u"include-tests"] == u"MRR": - hdr_param = u"Rec Rate" - else: - hdr_param = u"Thput" - - header.extend( - [ - f"{table[u'reference'][u'title']} {hdr_param} [Mpps]", - f"{table[u'reference'][u'title']} Stdev [Mpps]", - f"{table[u'compare'][u'title']} {hdr_param} [Mpps]", - f"{table[u'compare'][u'title']} Stdev [Mpps]", - u"Delta [%]" - ] + header = [ + u"Test Case", + f"{table[u'reference'][u'title']} " + f"Avg({table[u'include-tests']})", + f"{table[u'reference'][u'title']} " + f"Stdev({table[u'include-tests']})", + f"{table[u'compare'][u'title']} " + f"Avg({table[u'include-tests']})", + f"{table[u'compare'][u'title']} " + f"Stdev({table[u'include-tests']})", + f"Diff({table[u'reference'][u'title']}," + f"{table[u'compare'][u'title']})", + u"Stdev(Diff)" + ] + legend = ( + u"\nLegend:\n" + f"{table[u'reference'][u'title']} " + f"Avg({table[u'include-tests']}): " + f"Mean value of {table[u'include-tests']} [Mpps] computed from a " + f"series of runs of the listed tests executed using " + f"{table[u'reference'][u'title']} NIC.\n" + f"{table[u'reference'][u'title']} " + f"Stdev({table[u'include-tests']}): " + f"Standard deviation value of {table[u'include-tests']} [Mpps] " + f"computed from a series of runs of the listed tests executed " + f"using {table[u'reference'][u'title']} NIC.\n" + f"{table[u'compare'][u'title']} " + f"Avg({table[u'include-tests']}): " + f"Mean value of {table[u'include-tests']} [Mpps] computed from a " + f"series of runs of the listed tests executed using " + f"{table[u'compare'][u'title']} NIC.\n" + f"{table[u'compare'][u'title']} " + f"Stdev({table[u'include-tests']}): " + f"Standard deviation value of {table[u'include-tests']} [Mpps] " + f"computed from a series of runs of the listed tests executed " + f"using {table[u'compare'][u'title']} NIC.\n" + f"Diff({table[u'reference'][u'title']}," + f"{table[u'compare'][u'title']}): " + f"Percentage change calculated for mean values.\n" + u"Stdev(Diff): " + u"Standard deviation of percentage change calculated for mean " + u"values.\n" + u":END" ) except (AttributeError, KeyError) as err: @@ -930,18 +1438,18 @@ def table_nics_comparison(table, input_data): for job, builds in table[u"data"].items(): for build in builds: for tst_name, tst_data in data[job][str(build)].items(): - tst_name_mod = _tpc_modify_test_name(tst_name) + tst_name_mod = _tpc_modify_test_name(tst_name, ignore_nic=True) if tbl_dict.get(tst_name_mod, None) is None: - name = u"-".join(tst_data[u"name"].split(u"-")[:-1]) + name = tst_data[u'name'].rsplit(u'-', 1)[0] tbl_dict[tst_name_mod] = { u"name": name, u"ref-data": list(), u"cmp-data": list() } try: - result = None if table[u"include-tests"] == u"MRR": - result = tst_data[u"result"][u"receive-rate"] + result = (tst_data[u"result"][u"receive-rate"], + tst_data[u"result"][u"receive-stdev"]) elif table[u"include-tests"] == u"PDR": result = tst_data[u"throughput"][u"PDR"][u"LOWER"] elif table[u"include-tests"] == u"NDR": @@ -962,37 +1470,71 @@ def table_nics_comparison(table, input_data): tbl_lst = list() for tst_name in tbl_dict: item = [tbl_dict[tst_name][u"name"], ] - data_t = tbl_dict[tst_name][u"ref-data"] - if data_t: - item.append(round(mean(data_t) / 1000000, 2)) - item.append(round(stdev(data_t) / 1000000, 2)) + data_r = tbl_dict[tst_name][u"ref-data"] + if data_r: + if table[u"include-tests"] == u"MRR": + data_r_mean = data_r[0][0] + data_r_stdev = data_r[0][1] + else: + data_r_mean = mean(data_r) + data_r_stdev = stdev(data_r) + item.append(round(data_r_mean / 1e6, 1)) + item.append(round(data_r_stdev / 1e6, 1)) else: + data_r_mean = None + data_r_stdev = None item.extend([None, None]) - data_t = tbl_dict[tst_name][u"cmp-data"] - if data_t: - item.append(round(mean(data_t) / 1000000, 2)) - item.append(round(stdev(data_t) / 1000000, 2)) + data_c = tbl_dict[tst_name][u"cmp-data"] + if data_c: + if table[u"include-tests"] == u"MRR": + data_c_mean = data_c[0][0] + data_c_stdev = data_c[0][1] + else: + data_c_mean = mean(data_c) + data_c_stdev = stdev(data_c) + item.append(round(data_c_mean / 1e6, 1)) + item.append(round(data_c_stdev / 1e6, 1)) else: + data_c_mean = None + data_c_stdev = None item.extend([None, None]) - if item[-4] is not None and item[-2] is not None and item[-4] != 0: - item.append(int(relative_change(float(item[-4]), float(item[-2])))) - if len(item) == len(header): + if data_r_mean is not None and data_c_mean is not None: + delta, d_stdev = relative_change_stdev( + data_r_mean, data_c_mean, data_r_stdev, data_c_stdev + ) + try: + item.append(round(delta)) + except ValueError: + item.append(delta) + try: + item.append(round(d_stdev)) + except ValueError: + item.append(d_stdev) tbl_lst.append(item) # Sort the table according to the relative change tbl_lst.sort(key=lambda rel: rel[-1], reverse=True) # Generate csv tables: - with open(f"{table[u'output-file']}.csv", u"w") as file_handler: - file_handler.write(u",".join(header) + u"\n") + with open(f"{table[u'output-file']}.csv", u"wt") as file_handler: + file_handler.write(u";".join(header) + u"\n") for test in tbl_lst: - file_handler.write(u",".join([str(item) for item in test]) + u"\n") + file_handler.write(u";".join([str(item) for item in test]) + u"\n") convert_csv_to_pretty_txt(f"{table[u'output-file']}.csv", - f"{table[u'output-file']}.txt") + f"{table[u'output-file']}.txt", + delimiter=u";") + + with open(table[u'output-file'], u'a') as txt_file: + txt_file.write(legend) # Generate html table: - _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html") + _tpc_generate_html_table( + header, + tbl_lst, + table[u'output-file'], + legend=legend + ) def table_soak_vs_ndr(table, input_data): @@ -1017,14 +1559,37 @@ def table_soak_vs_ndr(table, input_data): # Prepare the header of the table try: header = [ - u"Test case", - f"{table[u'reference'][u'title']} Thput [Mpps]", - f"{table[u'reference'][u'title']} Stdev [Mpps]", - f"{table[u'compare'][u'title']} Thput [Mpps]", - f"{table[u'compare'][u'title']} Stdev [Mpps]", - u"Delta [%]", u"Stdev of delta [%]" + u"Test Case", + f"Avg({table[u'reference'][u'title']})", + f"Stdev({table[u'reference'][u'title']})", + f"Avg({table[u'compare'][u'title']})", + f"Stdev{table[u'compare'][u'title']})", + u"Diff", + u"Stdev(Diff)" ] - header_str = u",".join(header) + u"\n" + header_str = u";".join(header) + u"\n" + legend = ( + u"\nLegend:\n" + f"Avg({table[u'reference'][u'title']}): " + f"Mean value of {table[u'reference'][u'title']} [Mpps] computed " + f"from a series of runs of the listed tests.\n" + f"Stdev({table[u'reference'][u'title']}): " + f"Standard deviation value of {table[u'reference'][u'title']} " + f"[Mpps] computed from a series of runs of the listed tests.\n" + f"Avg({table[u'compare'][u'title']}): " + f"Mean value of {table[u'compare'][u'title']} [Mpps] computed from " + f"a series of runs of the listed tests.\n" + f"Stdev({table[u'compare'][u'title']}): " + f"Standard deviation value of {table[u'compare'][u'title']} [Mpps] " + f"computed from a series of runs of the listed tests.\n" + f"Diff({table[u'reference'][u'title']}," + f"{table[u'compare'][u'title']}): " + f"Percentage change calculated for mean values.\n" + u"Stdev(Diff): " + u"Standard deviation of percentage change calculated for mean " + u"values.\n" + u":END" + ) except (AttributeError, KeyError) as err: logging.error(f"The model is invalid, missing parameter: {repr(err)}") return @@ -1067,7 +1632,8 @@ def table_soak_vs_ndr(table, input_data): if tst_data[u"type"] not in (u"NDRPDR", u"MRR", u"BMRR"): continue if table[u"include-tests"] == u"MRR": - result = tst_data[u"result"][u"receive-rate"] + result = (tst_data[u"result"][u"receive-rate"], + tst_data[u"result"][u"receive-stdev"]) elif table[u"include-tests"] == u"PDR": result = \ tst_data[u"throughput"][u"PDR"][u"LOWER"] @@ -1087,29 +1653,43 @@ def table_soak_vs_ndr(table, input_data): item = [tbl_dict[tst_name][u"name"], ] data_r = tbl_dict[tst_name][u"ref-data"] if data_r: - data_r_mean = mean(data_r) - item.append(round(data_r_mean / 1000000, 2)) - data_r_stdev = stdev(data_r) - item.append(round(data_r_stdev / 1000000, 2)) + if table[u"include-tests"] == u"MRR": + data_r_mean = data_r[0][0] + data_r_stdev = data_r[0][1] + else: + data_r_mean = mean(data_r) + data_r_stdev = stdev(data_r) + item.append(round(data_r_mean / 1e6, 1)) + item.append(round(data_r_stdev / 1e6, 1)) else: data_r_mean = None data_r_stdev = None item.extend([None, None]) data_c = tbl_dict[tst_name][u"cmp-data"] if data_c: - data_c_mean = mean(data_c) - item.append(round(data_c_mean / 1000000, 2)) - data_c_stdev = stdev(data_c) - item.append(round(data_c_stdev / 1000000, 2)) + if table[u"include-tests"] == u"MRR": + data_c_mean = data_c[0][0] + data_c_stdev = data_c[0][1] + else: + data_c_mean = mean(data_c) + data_c_stdev = stdev(data_c) + item.append(round(data_c_mean / 1e6, 1)) + item.append(round(data_c_stdev / 1e6, 1)) else: data_c_mean = None data_c_stdev = None item.extend([None, None]) - if data_r_mean and data_c_mean: + if data_r_mean is not None and data_c_mean is not None: delta, d_stdev = relative_change_stdev( data_r_mean, data_c_mean, data_r_stdev, data_c_stdev) - item.append(round(delta, 2)) - item.append(round(d_stdev, 2)) + try: + item.append(round(delta)) + except ValueError: + item.append(delta) + try: + item.append(round(d_stdev)) + except ValueError: + item.append(d_stdev) tbl_lst.append(item) # Sort the table according to the relative change @@ -1117,15 +1697,24 @@ def table_soak_vs_ndr(table, input_data): # Generate csv tables: csv_file = f"{table[u'output-file']}.csv" - with open(csv_file, u"w") as file_handler: + with open(csv_file, u"wt") as file_handler: file_handler.write(header_str) for test in tbl_lst: - file_handler.write(u",".join([str(item) for item in test]) + u"\n") + file_handler.write(u";".join([str(item) for item in test]) + u"\n") - convert_csv_to_pretty_txt(csv_file, f"{table[u'output-file']}.txt") + convert_csv_to_pretty_txt( + csv_file, f"{table[u'output-file']}.txt", delimiter=u";" + ) + with open(f"{table[u'output-file']}.txt", u'a') as txt_file: + txt_file.write(legend) # Generate html table: - _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html") + _tpc_generate_html_table( + header, + tbl_lst, + table[u'output-file'], + legend=legend + ) def table_perf_trending_dash(table, input_data): @@ -1205,13 +1794,13 @@ def table_perf_trending_dash(table, input_data): rel_change_last = nan else: rel_change_last = round( - ((last_avg - avg_week_ago) / avg_week_ago) * 100, 2) + ((last_avg - avg_week_ago) / avg_week_ago) * 1e2, 2) if isnan(max_long_avg) or isnan(last_avg) or max_long_avg == 0.0: rel_change_long = nan else: rel_change_long = round( - ((last_avg - max_long_avg) / max_long_avg) * 100, 2) + ((last_avg - max_long_avg) / max_long_avg) * 1e2, 2) if classification_lst: if isnan(rel_change_last) and isnan(rel_change_long): @@ -1221,7 +1810,7 @@ def table_perf_trending_dash(table, input_data): continue tbl_lst.append( [tbl_dict[tst_name][u"name"], - round(last_avg / 1000000, 2), + round(last_avg / 1e6, 2), rel_change_last, rel_change_long, classification_lst[-win_size:].count(u"regression"), @@ -1240,7 +1829,7 @@ def table_perf_trending_dash(table, input_data): file_name = f"{table[u'output-file']}{table[u'output-file-ext']}" logging.info(f" Writing file: {file_name}") - with open(file_name, u"w") as file_handler: + with open(file_name, u"wt") as file_handler: file_handler.write(header_str) for test in tbl_sorted: file_handler.write(u",".join([str(item) for item in test]) + u'\n') @@ -1249,13 +1838,11 @@ def table_perf_trending_dash(table, input_data): convert_csv_to_pretty_txt(file_name, f"{table[u'output-file']}.txt") -def _generate_url(base, testbed, test_name): +def _generate_url(testbed, test_name): """Generate URL to a trending plot from the name of the test case. - :param base: The base part of URL common to all test cases. :param testbed: The testbed used for testing. :param test_name: The name of the test case. - :type base: str :type testbed: str :type test_name: str :returns: The URL to the plot with the trending data for the given test @@ -1263,140 +1850,140 @@ def _generate_url(base, testbed, test_name): :rtype str """ - url = base - file_name = u"" - anchor = u".html#" - feature = u"" - - if u"lbdpdk" in test_name or u"lbvpp" in test_name: - file_name = u"link_bonding" - - elif u"114b" in test_name and u"vhost" in test_name: - file_name = u"vts" - - elif u"testpmd" in test_name or u"l3fwd" in test_name: - file_name = u"dpdk" - - elif u"memif" in test_name: - file_name = u"container_memif" - feature = u"-base" - - elif u"srv6" in test_name: - file_name = u"srv6" - - elif u"vhost" in test_name: - if u"l2xcbase" in test_name or u"l2bdbasemaclrn" in test_name: - file_name = u"vm_vhost_l2" - if u"114b" in test_name: - feature = u"" - elif u"l2xcbase" in test_name and u"x520" in test_name: - feature = u"-base-l2xc" - elif u"l2bdbasemaclrn" in test_name and u"x520" in test_name: - feature = u"-base-l2bd" - else: - feature = u"-base" - elif u"ip4base" in test_name: - file_name = u"vm_vhost_ip4" - feature = u"-base" - - elif u"ipsecbasetnlsw" in test_name: - file_name = u"ipsecsw" - feature = u"-base-scale" - - elif u"ipsec" in test_name: - file_name = u"ipsec" - feature = u"-base-scale" - if u"hw-" in test_name: - file_name = u"ipsechw" - elif u"sw-" in test_name: - file_name = u"ipsecsw" - if u"-int-" in test_name: - feature = u"-base-scale-int" - elif u"tnl" in test_name: - feature = u"-base-scale-tnl" - - elif u"ethip4lispip" in test_name or u"ethip4vxlan" in test_name: - file_name = u"ip4_tunnels" - feature = u"-base" - - elif u"ip4base" in test_name or u"ip4scale" in test_name: - file_name = u"ip4" - if u"xl710" in test_name: - feature = u"-base-scale-features" - elif u"iacl" in test_name: - feature = u"-features-iacl" - elif u"oacl" in test_name: - feature = u"-features-oacl" - elif u"snat" in test_name or u"cop" in test_name: - feature = u"-features" - else: - feature = u"-base-scale" - - elif u"ip6base" in test_name or u"ip6scale" in test_name: - file_name = u"ip6" - feature = u"-base-scale" - - elif u"l2xcbase" in test_name or u"l2xcscale" in test_name \ - or u"l2bdbasemaclrn" in test_name or u"l2bdscale" in test_name: - file_name = u"l2" - if u"macip" in test_name: - feature = u"-features-macip" - elif u"iacl" in test_name: - feature = u"-features-iacl" - elif u"oacl" in test_name: - feature = u"-features-oacl" - else: - feature = u"-base-scale" - if u"x520" in test_name: - nic = u"x520-" + nic = u"x520" elif u"x710" in test_name: - nic = u"x710-" + nic = u"x710" elif u"xl710" in test_name: - nic = u"xl710-" + nic = u"xl710" elif u"xxv710" in test_name: - nic = u"xxv710-" + nic = u"xxv710" elif u"vic1227" in test_name: - nic = u"vic1227-" + nic = u"vic1227" elif u"vic1385" in test_name: - nic = u"vic1385-" + nic = u"vic1385" elif u"x553" in test_name: - nic = u"x553-" + nic = u"x553" + elif u"cx556" in test_name or u"cx556a" in test_name: + nic = u"cx556a" else: nic = u"" - anchor += nic if u"64b" in test_name: - framesize = u"64b" + frame_size = u"64b" elif u"78b" in test_name: - framesize = u"78b" + frame_size = u"78b" elif u"imix" in test_name: - framesize = u"imix" + frame_size = u"imix" elif u"9000b" in test_name: - framesize = u"9000b" + frame_size = u"9000b" elif u"1518b" in test_name: - framesize = u"1518b" + frame_size = u"1518b" elif u"114b" in test_name: - framesize = u"114b" + frame_size = u"114b" + else: + frame_size = u"" + + if u"1t1c" in test_name or \ + (u"-1c-" in test_name and + testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")): + cores = u"1t1c" + elif u"2t2c" in test_name or \ + (u"-2c-" in test_name and + testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")): + cores = u"2t2c" + elif u"4t4c" in test_name or \ + (u"-4c-" in test_name and + testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")): + cores = u"4t4c" + elif u"2t1c" in test_name or \ + (u"-1c-" in test_name and + testbed in (u"2n-skx", u"3n-skx", u"2n-clx")): + cores = u"2t1c" + elif u"4t2c" in test_name or \ + (u"-2c-" in test_name and + testbed in (u"2n-skx", u"3n-skx", u"2n-clx")): + cores = u"4t2c" + elif u"8t4c" in test_name or \ + (u"-4c-" in test_name and + testbed in (u"2n-skx", u"3n-skx", u"2n-clx")): + cores = u"8t4c" + else: + cores = u"" + + if u"testpmd" in test_name: + driver = u"testpmd" + elif u"l3fwd" in test_name: + driver = u"l3fwd" + elif u"avf" in test_name: + driver = u"avf" + elif u"rdma" in test_name: + driver = u"rdma" + elif u"dnv" in testbed or u"tsh" in testbed: + driver = u"ixgbe" + else: + driver = u"dpdk" + + if u"acl" in test_name or \ + u"macip" in test_name or \ + u"nat" in test_name or \ + u"policer" in test_name or \ + u"cop" in test_name: + bsf = u"features" + elif u"scale" in test_name: + bsf = u"scale" + elif u"base" in test_name: + bsf = u"base" + else: + bsf = u"base" + + if u"114b" in test_name and u"vhost" in test_name: + domain = u"vts" + elif u"testpmd" in test_name or u"l3fwd" in test_name: + domain = u"dpdk" + elif u"memif" in test_name: + domain = u"container_memif" + elif u"srv6" in test_name: + domain = u"srv6" + elif u"vhost" in test_name: + domain = u"vhost" + if u"vppl2xc" in test_name: + driver += u"-vpp" + else: + driver += u"-testpmd" + if u"lbvpplacp" in test_name: + bsf += u"-link-bonding" + elif u"ch" in test_name and u"vh" in test_name and u"vm" in test_name: + domain = u"nf_service_density_vnfc" + elif u"ch" in test_name and u"mif" in test_name and u"dcr" in test_name: + domain = u"nf_service_density_cnfc" + elif u"pl" in test_name and u"mif" in test_name and u"dcr" in test_name: + domain = u"nf_service_density_cnfp" + elif u"ipsec" in test_name: + domain = u"ipsec" + if u"sw" in test_name: + bsf += u"-sw" + elif u"hw" in test_name: + bsf += u"-hw" + elif u"ethip4vxlan" in test_name: + domain = u"ip4_tunnels" + elif u"ip4base" in test_name or u"ip4scale" in test_name: + domain = u"ip4" + elif u"ip6base" in test_name or u"ip6scale" in test_name: + domain = u"ip6" + elif u"l2xcbase" in test_name or \ + u"l2xcscale" in test_name or \ + u"l2bdbasemaclrn" in test_name or \ + u"l2bdscale" in test_name or \ + u"l2patch" in test_name: + domain = u"l2" else: - framesize = u"" - anchor += framesize + u"-" - - if u"1t1c" in test_name: - anchor += u"1t1c" - elif u"2t2c" in test_name: - anchor += u"2t2c" - elif u"4t4c" in test_name: - anchor += u"4t4c" - elif u"2t1c" in test_name: - anchor += u"2t1c" - elif u"4t2c" in test_name: - anchor += u"4t2c" - elif u"8t4c" in test_name: - anchor += u"8t4c" - - return url + file_name + u"-" + testbed + u"-" + nic + framesize + \ - feature.replace("-int", u"").replace("-tnl", u"") + anchor + feature + domain = u"" + + file_name = u"-".join((domain, testbed, nic)) + u".html#" + anchor_name = u"-".join((frame_size, cores, bsf, driver)) + + return file_name + anchor_name def table_perf_trending_dash_html(table, input_data): @@ -1483,11 +2070,8 @@ def table_perf_trending_dash_html(table, input_data): tdata, u"a", attrib=dict( - href=_generate_url( - u"../trending/", - table.get(u"testbed", None), - item - ) + href=f"../trending/" + f"{_generate_url(table.get(u'testbed', ''), item)}" ) ) ref.text = item @@ -1561,7 +2145,7 @@ def table_last_failed_tests(table, input_data): file_name = f"{table[u'output-file']}{table[u'output-file-ext']}" logging.info(f" Writing file: {file_name}") - with open(file_name, u"w") as file_handler: + with open(file_name, u"wt") as file_handler: for test in tbl_list: file_handler.write(test + u'\n') @@ -1666,7 +2250,7 @@ def table_failed_tests(table, input_data): file_name = f"{table[u'output-file']}{table[u'output-file-ext']}" logging.info(f" Writing file: {file_name}") - with open(file_name, u"w") as file_handler: + with open(file_name, u"wt") as file_handler: file_handler.write(u",".join(header) + u"\n") for test in tbl_sorted: file_handler.write(u",".join([str(item) for item in test]) + u'\n') @@ -1740,11 +2324,8 @@ def table_failed_tests_html(table, input_data): tdata, u"a", attrib=dict( - href=_generate_url( - u"../trending/", - table.get(u"testbed", None), - item - ) + href=f"../trending/" + f"{_generate_url(table.get(u'testbed', ''), item)}" ) ) ref.text = item @@ -1759,3 +2340,359 @@ def table_failed_tests_html(table, input_data): except KeyError: logging.warning(u"The output file is not defined.") return + + +def table_comparison(table, input_data): + """Generate the table(s) with algorithm: table_comparison + specified in the specification file. + + :param table: Table to generate. + :param input_data: Data to process. + :type table: pandas.Series + :type input_data: InputData + """ + logging.info(f" Generating the table {table.get(u'title', u'')} ...") + + # Transform the data + logging.info( + f" Creating the data set for the {table.get(u'type', u'')} " + f"{table.get(u'title', u'')}." + ) + + columns = table.get(u"columns", None) + if not columns: + logging.error( + f"No columns specified for {table.get(u'title', u'')}. Skipping." + ) + return + + cols = list() + for idx, col in enumerate(columns): + if col.get(u"data", None) is None: + logging.warning(f"No data for column {col.get(u'title', u'')}") + continue + data = input_data.filter_data( + table, + params=[u"throughput", u"result", u"name", u"parent", u"tags"], + data=col[u"data"], + continue_on_error=True + ) + col_data = { + u"title": col.get(u"title", f"Column{idx}"), + u"data": dict() + } + for builds in data.values: + for build in builds: + for tst_name, tst_data in build.items(): + tst_name_mod = \ + _tpc_modify_test_name(tst_name).replace(u"2n1l-", u"") + if col_data[u"data"].get(tst_name_mod, None) is None: + name = tst_data[u'name'].rsplit(u'-', 1)[0] + if u"across testbeds" in table[u"title"].lower() or \ + u"across topologies" in table[u"title"].lower(): + name = _tpc_modify_displayed_test_name(name) + col_data[u"data"][tst_name_mod] = { + u"name": name, + u"replace": True, + u"data": list(), + u"mean": None, + u"stdev": None + } + _tpc_insert_data( + target=col_data[u"data"][tst_name_mod][u"data"], + src=tst_data, + include_tests=table[u"include-tests"] + ) + + replacement = col.get(u"data-replacement", None) + if replacement: + rpl_data = input_data.filter_data( + table, + params=[u"throughput", u"result", u"name", u"parent", u"tags"], + data=replacement, + continue_on_error=True + ) + for builds in rpl_data.values: + for build in builds: + for tst_name, tst_data in build.items(): + tst_name_mod = \ + _tpc_modify_test_name(tst_name).\ + replace(u"2n1l-", u"") + if col_data[u"data"].get(tst_name_mod, None) is None: + name = tst_data[u'name'].rsplit(u'-', 1)[0] + if u"across testbeds" in table[u"title"].lower() \ + or u"across topologies" in \ + table[u"title"].lower(): + name = _tpc_modify_displayed_test_name(name) + col_data[u"data"][tst_name_mod] = { + u"name": name, + u"replace": False, + u"data": list(), + u"mean": None, + u"stdev": None + } + if col_data[u"data"][tst_name_mod][u"replace"]: + col_data[u"data"][tst_name_mod][u"replace"] = False + col_data[u"data"][tst_name_mod][u"data"] = list() + _tpc_insert_data( + target=col_data[u"data"][tst_name_mod][u"data"], + src=tst_data, + include_tests=table[u"include-tests"] + ) + + if table[u"include-tests"] in (u"NDR", u"PDR"): + for tst_name, tst_data in col_data[u"data"].items(): + if tst_data[u"data"]: + tst_data[u"mean"] = mean(tst_data[u"data"]) + tst_data[u"stdev"] = stdev(tst_data[u"data"]) + elif table[u"include-tests"] in (u"MRR", ): + for tst_name, tst_data in col_data[u"data"].items(): + if tst_data[u"data"]: + tst_data[u"mean"] = tst_data[u"data"][0] + tst_data[u"stdev"] = tst_data[u"data"][0] + + cols.append(col_data) + + tbl_dict = dict() + for col in cols: + for tst_name, tst_data in col[u"data"].items(): + if tbl_dict.get(tst_name, None) is None: + tbl_dict[tst_name] = { + "name": tst_data[u"name"] + } + tbl_dict[tst_name][col[u"title"]] = { + u"mean": tst_data[u"mean"], + u"stdev": tst_data[u"stdev"] + } + + tbl_lst = list() + for tst_data in tbl_dict.values(): + row = [tst_data[u"name"], ] + for col in cols: + row.append(tst_data.get(col[u"title"], None)) + tbl_lst.append(row) + + comparisons = table.get(u"comparisons", None) + if comparisons and isinstance(comparisons, list): + for idx, comp in enumerate(comparisons): + try: + col_ref = int(comp[u"reference"]) + col_cmp = int(comp[u"compare"]) + except KeyError: + logging.warning(u"Comparison: No references defined! Skipping.") + comparisons.pop(idx) + continue + if not (0 < col_ref <= len(cols) and + 0 < col_cmp <= len(cols)) or \ + col_ref == col_cmp: + logging.warning(f"Wrong values of reference={col_ref} " + f"and/or compare={col_cmp}. Skipping.") + comparisons.pop(idx) + continue + + tbl_cmp_lst = list() + if comparisons: + for row in tbl_lst: + new_row = deepcopy(row) + add_to_tbl = False + for comp in comparisons: + ref_itm = row[int(comp[u"reference"])] + if ref_itm is None and \ + comp.get(u"reference-alt", None) is not None: + ref_itm = row[int(comp[u"reference-alt"])] + cmp_itm = row[int(comp[u"compare"])] + if ref_itm is not None and cmp_itm is not None and \ + ref_itm[u"mean"] is not None and \ + cmp_itm[u"mean"] is not None and \ + ref_itm[u"stdev"] is not None and \ + cmp_itm[u"stdev"] is not None: + delta, d_stdev = relative_change_stdev( + ref_itm[u"mean"], cmp_itm[u"mean"], + ref_itm[u"stdev"], cmp_itm[u"stdev"] + ) + new_row.append( + { + u"mean": delta * 1e6, + u"stdev": d_stdev * 1e6 + } + ) + add_to_tbl = True + else: + new_row.append(None) + if add_to_tbl: + tbl_cmp_lst.append(new_row) + + tbl_cmp_lst.sort(key=lambda rel: rel[0], reverse=False) + tbl_cmp_lst.sort(key=lambda rel: rel[-1][u'mean'], reverse=True) + + rcas = list() + rca_in = table.get(u"rca", None) + if rca_in and isinstance(rca_in, list): + for idx, itm in enumerate(rca_in): + try: + with open(itm.get(u"data", u""), u"r") as rca_file: + rcas.append( + { + u"title": itm.get(u"title", f"RCA{idx}"), + u"data": load(rca_file, Loader=FullLoader) + } + ) + except (YAMLError, IOError) as err: + logging.warning( + f"The RCA file {itm.get(u'data', u'')} does not exist or " + f"it is corrupted!" + ) + logging.debug(repr(err)) + + tbl_for_csv = list() + for line in tbl_cmp_lst: + + row = [line[0], ] + + for idx, rca in enumerate(rcas): + rca_nr = rca[u"data"].get(row[0 + idx], u"-") + row.insert(idx, f"[{rca_nr}]" if rca_nr != u"-" else u"-") + + for idx, itm in enumerate(line[1:]): + if itm is None: + row.append(u"NT") + row.append(u"NT") + else: + row.append(round(float(itm[u'mean']) / 1e6, 3)) + row.append(round(float(itm[u'stdev']) / 1e6, 3)) + tbl_for_csv.append(row) + + header_csv = [rca[u"title"] for rca in rcas] + header_csv.append(u"Test Case") + for col in cols: + header_csv.append(f"Avg({col[u'title']})") + header_csv.append(f"Stdev({col[u'title']})") + for comp in comparisons: + header_csv.append( + f"Avg({cols[comp[u'reference'] - 1][u'title']}," + f"{cols[comp[u'compare'] - 1][u'title']})" + ) + header_csv.append( + f"Stdev({cols[comp[u'reference'] - 1][u'title']}," + f"{cols[comp[u'compare'] - 1][u'title']})" + ) + + csv_file = f"{table[u'output-file']}-csv.csv" + with open(csv_file, u"wt", encoding='utf-8') as file_handler: + file_handler.write(u";".join(header_csv) + u"\n") + for test in tbl_for_csv: + file_handler.write(u";".join([str(item) for item in test]) + u"\n") + + tbl_final = list() + for line in tbl_cmp_lst: + row = [line[0], ] + for idx, rca in enumerate(rcas): + rca_nr = rca[u"data"].get(row[0 + idx], u"-") + row.insert(idx, f"[{rca_nr}]" if rca_nr != u"-" else u"-") + for idx, itm in enumerate(line[1:]): + if itm is None: + row.append(u"NT") + else: + if idx < len(cols): + row.append( + f"{round(float(itm[u'mean']) / 1e6, 1)} " + f"\u00B1{round(float(itm[u'stdev']) / 1e6, 1)}". + replace(u"nan", u"NaN") + ) + else: + row.append( + f"{round(float(itm[u'mean']) / 1e6, 1):+} " + f"\u00B1{round(float(itm[u'stdev']) / 1e6, 1)}". + replace(u"nan", u"NaN") + ) + tbl_final.append(row) + + header = [rca[u"title"] for rca in rcas] + header.append(u"Test Case") + header.extend([col[u"title"] for col in cols]) + header.extend( + [f"Diff({cols[comp[u'reference'] - 1][u'title']}," + f"{cols[comp[u'compare'] - 1][u'title']})" + for comp in comparisons] + ) + + # Generate csv tables: + csv_file = f"{table[u'output-file']}.csv" + with open(csv_file, u"wt", encoding='utf-8') as file_handler: + file_handler.write(u";".join(header) + u"\n") + for test in tbl_final: + file_handler.write(u";".join([str(item) for item in test]) + u"\n") + + # Generate txt table: + txt_file_name = f"{table[u'output-file']}.txt" + convert_csv_to_pretty_txt(csv_file, txt_file_name, delimiter=u";") + + # Generate rst table: + file_name = table[u'output-file'].split(u"/")[-1] + if u"vpp" in table[u'output-file']: + path = u"_tmp/src/vpp_performance_tests/comparisons/" + else: + path = u"_tmp/src/dpdk_performance_tests/comparisons/" + rst_file_name = f"{path}{file_name}-txt.rst" + csv_file_name = f"{path}{file_name}.csv" + with open(csv_file_name, u"wt", encoding='utf-8') as file_handler: + file_handler.write( + u",".join( + [f'"{itm}"' for itm in header] + ) + u"\n" + ) + for test in tbl_final: + file_handler.write( + u",".join( + [f'"{itm}"' for itm in test] + ) + u"\n" + ) + + convert_csv_to_pretty_txt(csv_file_name, rst_file_name, delimiter=u",") + + legend = u"\nLegend:\n" + for idx, rca in enumerate(rcas): + try: + desc = ( + f"Diff({cols[comparisons[idx][u'reference'] - 1][u'title']}," + f"{cols[comparisons[idx][u'compare'] - 1][u'title']})\n" + ) + except (KeyError, IndexError): + desc = u"\n" + legend += f"{rca[u'title']}: Root Cause Analysis for {desc}" + legend += ( + u"First part of the result is a mean value [Mpps].\n" + f"Second part of the result following '\u00B1' is a standard " + u"deviation [Mpps].\n" + u"First part of Diff is a relative change of mean values [%].\n" + f"Second part of Diff following '\u00B1' is a standard deviation " + u"of the Diff [percentual points].\n" + u"NT: Not tested.\n" + ) + + footnote = u"" + for rca in rcas: + footnote += f"\n{rca[u'title']}:\n" + footnote += rca[u"data"].get(u"footnote", u"") + + with open(txt_file_name, u'a', encoding='utf-8') as txt_file: + txt_file.write(legend) + if footnote: + txt_file.write(footnote) + txt_file.write(u":END") + + with open(rst_file_name, u'a', encoding='utf-8') as txt_file: + txt_file.write(legend.replace(u"\n", u" |br| ")) + if footnote: + txt_file.write(footnote.replace(u"\n", u" |br| ")) + txt_file.write(u":END") + + # Generate html table: + _tpc_generate_html_table( + header, + tbl_final, + table[u'output-file'], + legend=legend, + footnote=footnote, + sort_data=False + )