X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=blobdiff_plain;f=resources%2Ftools%2Fpresentation%2Fgenerator_tables.py;h=4cbc7c074647b0706d95694de2689ac4c08956e0;hp=606ca661ee65d7cc49cd45b30c7367afcfaecce2;hb=b95d8570e5edb080d7cfc001daed66b47811068a;hpb=4be26a2f7f0cf0539e198864d69bd34bd278dda4 diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py index 606ca661ee..4cbc7c0746 100644 --- a/resources/tools/presentation/generator_tables.py +++ b/resources/tools/presentation/generator_tables.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019 Cisco and/or its affiliates. +# Copyright (c) 2020 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -19,18 +19,23 @@ import logging import csv import re -from string import replace from collections import OrderedDict -from numpy import nan, isnan from xml.etree import ElementTree as ET from datetime import datetime as dt from datetime import timedelta -from utils import mean, stdev, relative_change, classify_anomalies, \ +import plotly.graph_objects as go +import plotly.offline as ploff +import pandas as pd + +from numpy import nan, isnan +from yaml import load, FullLoader, YAMLError + +from pal_utils import mean, stdev, classify_anomalies, \ convert_csv_to_pretty_txt, relative_change_stdev -REGEX_NIC = re.compile(r'\d*ge\dp\d\D*\d*') +REGEX_NIC = re.compile(r'(\d*ge\dp\d\D*\d*[a-z]*)') def generate_tables(spec, data): @@ -42,18 +47,34 @@ def generate_tables(spec, data): :type data: InputData """ - logging.info("Generating the tables ...") + generator = { + u"table_merged_details": table_merged_details, + u"table_perf_comparison": table_perf_comparison, + u"table_perf_comparison_nic": table_perf_comparison_nic, + u"table_nics_comparison": table_nics_comparison, + u"table_soak_vs_ndr": table_soak_vs_ndr, + u"table_perf_trending_dash": table_perf_trending_dash, + u"table_perf_trending_dash_html": table_perf_trending_dash_html, + u"table_last_failed_tests": table_last_failed_tests, + u"table_failed_tests": table_failed_tests, + u"table_failed_tests_html": table_failed_tests_html, + u"table_oper_data_html": table_oper_data_html + } + + logging.info(u"Generating the tables ...") for table in spec.tables: try: - eval(table["algorithm"])(table, data) + generator[table[u"algorithm"]](table, data) except NameError as err: - logging.error("Probably algorithm '{alg}' is not defined: {err}". - format(alg=table["algorithm"], err=repr(err))) - logging.info("Done.") + logging.error( + f"Probably algorithm {table[u'algorithm']} is not defined: " + f"{repr(err)}" + ) + logging.info(u"Done.") -def table_details(table, input_data): - """Generate the table(s) with algorithm: table_detailed_test_results +def table_oper_data_html(table, input_data): + """Generate the table(s) with algorithm: html_table_oper_data specified in the specification file. :param table: Table to generate. @@ -62,62 +83,198 @@ def table_details(table, input_data): :type input_data: InputData """ - logging.info(" Generating the table {0} ...". - format(table.get("title", ""))) - + logging.info(f" Generating the table {table.get(u'title', u'')} ...") # Transform the data - logging.info(" Creating the data set for the {0} '{1}'.". - format(table.get("type", ""), table.get("title", ""))) - data = input_data.filter_data(table) - - # Prepare the header of the tables - header = list() - for column in table["columns"]: - header.append('"{0}"'.format(str(column["title"]).replace('"', '""'))) - - # Generate the data for the table according to the model in the table - # specification - job = table["data"].keys()[0] - build = str(table["data"][job][0]) - try: - suites = input_data.suites(job, build) - except KeyError: - logging.error(" No data available. The table will not be generated.") + logging.info( + f" Creating the data set for the {table.get(u'type', u'')} " + f"{table.get(u'title', u'')}." + ) + data = input_data.filter_data( + table, + params=[u"name", u"parent", u"show-run", u"type"], + continue_on_error=True + ) + if data.empty: return + data = input_data.merge_data(data) - for suite_longname, suite in suites.iteritems(): - # Generate data - suite_name = suite["name"] - table_lst = list() - for test in data[job][build].keys(): - if data[job][build][test]["parent"] in suite_name: - row_lst = list() - for column in table["columns"]: - try: - col_data = str(data[job][build][test][column["data"]. - split(" ")[1]]).replace('"', '""') - if column["data"].split(" ")[1] in ("conf-history", - "show-run"): - col_data = replace(col_data, " |br| ", "", - maxreplace=1) - col_data = " |prein| {0} |preout| ".\ - format(col_data[:-5]) - row_lst.append('"{0}"'.format(col_data)) - except KeyError: - row_lst.append("No data") - table_lst.append(row_lst) + sort_tests = table.get(u"sort", None) + if sort_tests: + args = dict( + inplace=True, + ascending=(sort_tests == u"ascending") + ) + data.sort_index(**args) + + suites = input_data.filter_data( + table, + continue_on_error=True, + data_set=u"suites" + ) + if suites.empty: + return + suites = input_data.merge_data(suites) - # Write the data to file - if table_lst: - file_name = "{0}_{1}{2}".format(table["output-file"], suite_name, - table["output-file-ext"]) - logging.info(" Writing file: '{}'".format(file_name)) - with open(file_name, "w") as file_handler: - file_handler.write(",".join(header) + "\n") - for item in table_lst: - file_handler.write(",".join(item) + "\n") + def _generate_html_table(tst_data): + """Generate an HTML table with operational data for the given test. + + :param tst_data: Test data to be used to generate the table. + :type tst_data: pandas.Series + :returns: HTML table with operational data. + :rtype: str + """ + + colors = { + u"header": u"#7eade7", + u"empty": u"#ffffff", + u"body": (u"#e9f1fb", u"#d4e4f7") + } + + tbl = ET.Element(u"table", attrib=dict(width=u"100%", border=u"0")) + + trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])) + thead = ET.SubElement( + trow, u"th", attrib=dict(align=u"left", colspan=u"6") + ) + thead.text = tst_data[u"name"] + + trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])) + thead = ET.SubElement( + trow, u"th", attrib=dict(align=u"left", colspan=u"6") + ) + thead.text = u"\t" + + if tst_data.get(u"show-run", u"No Data") == u"No Data": + trow = ET.SubElement( + tbl, u"tr", attrib=dict(bgcolor=colors[u"header"]) + ) + tcol = ET.SubElement( + trow, u"td", attrib=dict(align=u"left", colspan=u"6") + ) + tcol.text = u"No Data" + + trow = ET.SubElement( + tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]) + ) + thead = ET.SubElement( + trow, u"th", attrib=dict(align=u"left", colspan=u"6") + ) + font = ET.SubElement( + thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff") + ) + font.text = u"." + return str(ET.tostring(tbl, encoding=u"unicode")) + + tbl_hdr = ( + u"Name", + u"Nr of Vectors", + u"Nr of Packets", + u"Suspends", + u"Cycles per Packet", + u"Average Vector Size" + ) + + for dut_data in tst_data[u"show-run"].values(): + trow = ET.SubElement( + tbl, u"tr", attrib=dict(bgcolor=colors[u"header"]) + ) + tcol = ET.SubElement( + trow, u"td", attrib=dict(align=u"left", colspan=u"6") + ) + if dut_data.get(u"threads", None) is None: + tcol.text = u"No Data" + continue - logging.info(" Done.") + bold = ET.SubElement(tcol, u"b") + bold.text = ( + f"Host IP: {dut_data.get(u'host', '')}, " + f"Socket: {dut_data.get(u'socket', '')}" + ) + trow = ET.SubElement( + tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]) + ) + thead = ET.SubElement( + trow, u"th", attrib=dict(align=u"left", colspan=u"6") + ) + thead.text = u"\t" + + for thread_nr, thread in dut_data[u"threads"].items(): + trow = ET.SubElement( + tbl, u"tr", attrib=dict(bgcolor=colors[u"header"]) + ) + tcol = ET.SubElement( + trow, u"td", attrib=dict(align=u"left", colspan=u"6") + ) + bold = ET.SubElement(tcol, u"b") + bold.text = u"main" if thread_nr == 0 else f"worker_{thread_nr}" + trow = ET.SubElement( + tbl, u"tr", attrib=dict(bgcolor=colors[u"header"]) + ) + for idx, col in enumerate(tbl_hdr): + tcol = ET.SubElement( + trow, u"td", + attrib=dict(align=u"right" if idx else u"left") + ) + font = ET.SubElement( + tcol, u"font", attrib=dict(size=u"2") + ) + bold = ET.SubElement(font, u"b") + bold.text = col + for row_nr, row in enumerate(thread): + trow = ET.SubElement( + tbl, u"tr", + attrib=dict(bgcolor=colors[u"body"][row_nr % 2]) + ) + for idx, col in enumerate(row): + tcol = ET.SubElement( + trow, u"td", + attrib=dict(align=u"right" if idx else u"left") + ) + font = ET.SubElement( + tcol, u"font", attrib=dict(size=u"2") + ) + if isinstance(col, float): + font.text = f"{col:.2f}" + else: + font.text = str(col) + trow = ET.SubElement( + tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]) + ) + thead = ET.SubElement( + trow, u"th", attrib=dict(align=u"left", colspan=u"6") + ) + thead.text = u"\t" + + trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])) + thead = ET.SubElement( + trow, u"th", attrib=dict(align=u"left", colspan=u"6") + ) + font = ET.SubElement( + thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff") + ) + font.text = u"." + + return str(ET.tostring(tbl, encoding=u"unicode")) + + for suite in suites.values: + html_table = str() + for test_data in data.values: + if test_data[u"parent"] not in suite[u"name"]: + continue + html_table += _generate_html_table(test_data) + if not html_table: + continue + try: + file_name = f"{table[u'output-file']}{suite[u'name']}.rst" + with open(f"{file_name}", u'w') as html_file: + logging.info(f" Writing file: {file_name}") + html_file.write(u".. raw:: html\n\n\t") + html_file.write(html_table) + html_file.write(u"\n\t



\n") + except KeyError: + logging.warning(u"The output file is not defined.") + return + logging.info(u" Done.") def table_merged_details(table, input_data): @@ -130,65 +287,307 @@ def table_merged_details(table, input_data): :type input_data: InputData """ - logging.info(" Generating the table {0} ...". - format(table.get("title", ""))) + logging.info(f" Generating the table {table.get(u'title', u'')} ...") # Transform the data - logging.info(" Creating the data set for the {0} '{1}'.". - format(table.get("type", ""), table.get("title", ""))) - data = input_data.filter_data(table) + logging.info( + f" Creating the data set for the {table.get(u'type', u'')} " + f"{table.get(u'title', u'')}." + ) + data = input_data.filter_data(table, continue_on_error=True) data = input_data.merge_data(data) - data.sort_index(inplace=True) - logging.info(" Creating the data set for the {0} '{1}'.". - format(table.get("type", ""), table.get("title", ""))) - suites = input_data.filter_data(table, data_set="suites") + sort_tests = table.get(u"sort", None) + if sort_tests: + args = dict( + inplace=True, + ascending=(sort_tests == u"ascending") + ) + data.sort_index(**args) + + suites = input_data.filter_data( + table, continue_on_error=True, data_set=u"suites") suites = input_data.merge_data(suites) # Prepare the header of the tables header = list() - for column in table["columns"]: - header.append('"{0}"'.format(str(column["title"]).replace('"', '""'))) + for column in table[u"columns"]: + header.append( + u'"{0}"'.format(str(column[u"title"]).replace(u'"', u'""')) + ) - for _, suite in suites.iteritems(): + for suite in suites.values: # Generate data - suite_name = suite["name"] + suite_name = suite[u"name"] table_lst = list() for test in data.keys(): - if data[test]["parent"] in suite_name: - row_lst = list() - for column in table["columns"]: - try: - col_data = str(data[test][column["data"]. - split(" ")[1]]).replace('"', '""') - col_data = replace(col_data, "No Data", - "Not Captured ") - if column["data"].split(" ")[1] in ("conf-history", - "show-run"): - col_data = replace(col_data, " |br| ", "", - maxreplace=1) - col_data = " |prein| {0} |preout| ".\ - format(col_data[:-5]) - row_lst.append('"{0}"'.format(col_data)) - except KeyError: - row_lst.append('"Not captured"') + if data[test][u"parent"] not in suite_name: + continue + row_lst = list() + for column in table[u"columns"]: + try: + col_data = str(data[test][column[ + u"data"].split(u" ")[1]]).replace(u'"', u'""') + # Do not include tests with "Test Failed" in test message + if u"Test Failed" in col_data: + continue + col_data = col_data.replace( + u"No Data", u"Not Captured " + ) + if column[u"data"].split(u" ")[1] in (u"name", ): + if len(col_data) > 30: + col_data_lst = col_data.split(u"-") + half = int(len(col_data_lst) / 2) + col_data = f"{u'-'.join(col_data_lst[:half])}" \ + f"- |br| " \ + f"{u'-'.join(col_data_lst[half:])}" + col_data = f" |prein| {col_data} |preout| " + elif column[u"data"].split(u" ")[1] in (u"msg", ): + # Temporary solution: remove NDR results from message: + if bool(table.get(u'remove-ndr', False)): + try: + col_data = col_data.split(u" |br| ", 1)[1] + except IndexError: + pass + col_data = f" |prein| {col_data} |preout| " + elif column[u"data"].split(u" ")[1] in \ + (u"conf-history", u"show-run"): + col_data = col_data.replace(u" |br| ", u"", 1) + col_data = f" |prein| {col_data[:-5]} |preout| " + row_lst.append(f'"{col_data}"') + except KeyError: + row_lst.append(u'"Not captured"') + if len(row_lst) == len(table[u"columns"]): table_lst.append(row_lst) # Write the data to file if table_lst: - file_name = "{0}_{1}{2}".format(table["output-file"], suite_name, - table["output-file-ext"]) - logging.info(" Writing file: '{}'".format(file_name)) - with open(file_name, "w") as file_handler: - file_handler.write(",".join(header) + "\n") + separator = u"" if table[u'output-file'].endswith(u"/") else u"_" + file_name = f"{table[u'output-file']}{separator}{suite_name}.csv" + logging.info(f" Writing file: {file_name}") + with open(file_name, u"wt") as file_handler: + file_handler.write(u",".join(header) + u"\n") for item in table_lst: - file_handler.write(",".join(item) + "\n") + file_handler.write(u",".join(item) + u"\n") - logging.info(" Done.") + logging.info(u" Done.") -def table_performance_comparison(table, input_data): - """Generate the table(s) with algorithm: table_performance_comparison +def _tpc_modify_test_name(test_name): + """Modify a test name by replacing its parts. + + :param test_name: Test name to be modified. + :type test_name: str + :returns: Modified test name. + :rtype: str + """ + test_name_mod = test_name.\ + replace(u"-ndrpdrdisc", u""). \ + replace(u"-ndrpdr", u"").\ + replace(u"-pdrdisc", u""). \ + replace(u"-ndrdisc", u"").\ + replace(u"-pdr", u""). \ + replace(u"-ndr", u""). \ + replace(u"1t1c", u"1c").\ + replace(u"2t1c", u"1c"). \ + replace(u"2t2c", u"2c").\ + replace(u"4t2c", u"2c"). \ + replace(u"4t4c", u"4c").\ + replace(u"8t4c", u"4c") + + return re.sub(REGEX_NIC, u"", test_name_mod) + + +def _tpc_modify_displayed_test_name(test_name): + """Modify a test name which is displayed in a table by replacing its parts. + + :param test_name: Test name to be modified. + :type test_name: str + :returns: Modified test name. + :rtype: str + """ + return test_name.\ + replace(u"1t1c", u"1c").\ + replace(u"2t1c", u"1c"). \ + replace(u"2t2c", u"2c").\ + replace(u"4t2c", u"2c"). \ + replace(u"4t4c", u"4c").\ + replace(u"8t4c", u"4c") + + +def _tpc_insert_data(target, src, include_tests): + """Insert src data to the target structure. + + :param target: Target structure where the data is placed. + :param src: Source data to be placed into the target stucture. + :param include_tests: Which results will be included (MRR, NDR, PDR). + :type target: list + :type src: dict + :type include_tests: str + """ + try: + if include_tests == u"MRR": + target.append( + ( + src[u"result"][u"receive-rate"], + src[u"result"][u"receive-stdev"] + ) + ) + elif include_tests == u"PDR": + target.append(src[u"throughput"][u"PDR"][u"LOWER"]) + elif include_tests == u"NDR": + target.append(src[u"throughput"][u"NDR"][u"LOWER"]) + except (KeyError, TypeError): + pass + + +def _tpc_sort_table(table): + """Sort the table this way: + + 1. Put "New in CSIT-XXXX" at the first place. + 2. Put "See footnote" at the second place. + 3. Sort the rest by "Delta". + + :param table: Table to sort. + :type table: list + :returns: Sorted table. + :rtype: list + """ + + tbl_new = list() + tbl_see = list() + tbl_delta = list() + for item in table: + if isinstance(item[-1], str): + if u"New in CSIT" in item[-1]: + tbl_new.append(item) + elif u"See footnote" in item[-1]: + tbl_see.append(item) + else: + tbl_delta.append(item) + + # Sort the tables: + tbl_new.sort(key=lambda rel: rel[0], reverse=False) + tbl_see.sort(key=lambda rel: rel[0], reverse=False) + tbl_see.sort(key=lambda rel: rel[-2], reverse=False) + tbl_delta.sort(key=lambda rel: rel[0], reverse=False) + tbl_delta.sort(key=lambda rel: rel[-2], reverse=True) + + # Put the tables together: + table = list() + # We do not want "New in CSIT": + # table.extend(tbl_new) + table.extend(tbl_see) + table.extend(tbl_delta) + + return table + + +def _tpc_generate_html_table(header, data, output_file_name): + """Generate html table from input data with simple sorting possibility. + + :param header: Table header. + :param data: Input data to be included in the table. It is a list of lists. + Inner lists are rows in the table. All inner lists must be of the same + length. The length of these lists must be the same as the length of the + header. + :param output_file_name: The name (relative or full path) where the + generated html table is written. + :type header: list + :type data: list of lists + :type output_file_name: str + """ + + try: + idx = header.index(u"Test case") + except ValueError: + idx = 0 + params = { + u"align-hdr": ([u"left", u"center"], [u"left", u"left", u"center"]), + u"align-itm": ([u"left", u"right"], [u"left", u"left", u"right"]), + u"width": ([28, 9], [4, 24, 10]) + } + + df_data = pd.DataFrame(data, columns=header) + + df_sorted = [df_data.sort_values( + by=[key, header[idx]], ascending=[True, True] + if key != header[idx] else [False, True]) for key in header] + df_sorted_rev = [df_data.sort_values( + by=[key, header[idx]], ascending=[False, True] + if key != header[idx] else [True, True]) for key in header] + df_sorted.extend(df_sorted_rev) + + fill_color = [[u"#d4e4f7" if idx % 2 else u"#e9f1fb" + for idx in range(len(df_data))]] + table_header = dict( + values=[f"{item}" for item in header], + fill_color=u"#7eade7", + align=params[u"align-hdr"][idx] + ) + + fig = go.Figure() + + for table in df_sorted: + columns = [table.get(col) for col in header] + fig.add_trace( + go.Table( + columnwidth=params[u"width"][idx], + header=table_header, + cells=dict( + values=columns, + fill_color=fill_color, + align=params[u"align-itm"][idx] + ) + ) + ) + + buttons = list() + menu_items = [f"{itm} (ascending)" for itm in header] + menu_items_rev = [f"{itm} (descending)" for itm in header] + menu_items.extend(menu_items_rev) + for idx, hdr in enumerate(menu_items): + visible = [False, ] * len(menu_items) + visible[idx] = True + buttons.append( + dict( + label=hdr.replace(u" [Mpps]", u""), + method=u"update", + args=[{u"visible": visible}], + ) + ) + + fig.update_layout( + updatemenus=[ + go.layout.Updatemenu( + type=u"dropdown", + direction=u"down", + x=0.03, + xanchor=u"left", + y=1.045, + yanchor=u"top", + active=len(menu_items) - 1, + buttons=list(buttons) + ) + ], + annotations=[ + go.layout.Annotation( + text=u"Sort by:", + x=0, + xref=u"paper", + y=1.035, + yref=u"paper", + align=u"left", + showarrow=False + ) + ] + ) + + ploff.plot(fig, show_link=False, auto_open=False, filename=output_file_name) + + +def table_perf_comparison(table, input_data): + """Generate the table(s) with algorithm: table_perf_comparison specified in the specification file. :param table: Table to generate. @@ -197,262 +596,303 @@ def table_performance_comparison(table, input_data): :type input_data: InputData """ - logging.info(" Generating the table {0} ...". - format(table.get("title", ""))) + logging.info(f" Generating the table {table.get(u'title', u'')} ...") # Transform the data - logging.info(" Creating the data set for the {0} '{1}'.". - format(table.get("type", ""), table.get("title", ""))) + logging.info( + f" Creating the data set for the {table.get(u'type', u'')} " + f"{table.get(u'title', u'')}." + ) data = input_data.filter_data(table, continue_on_error=True) # Prepare the header of the tables try: - header = ["Test case", ] + header = [u"Test case", ] - if table["include-tests"] == "MRR": - hdr_param = "Rec Rate" + rca_data = None + rca = table.get(u"rca", None) + if rca: + try: + with open(rca.get(u"data-file", ""), u"r") as rca_file: + rca_data = load(rca_file, Loader=FullLoader) + header.insert(0, rca.get(u"title", "RCA")) + except (YAMLError, IOError) as err: + logging.warning(repr(err)) + + if table[u"include-tests"] == u"MRR": + hdr_param = u"Rec Rate" else: - hdr_param = "Thput" + hdr_param = u"Thput" - history = table.get("history", None) - if history: - for item in history: - header.extend( - ["{0} {1} [Mpps]".format(item["title"], hdr_param), - "{0} Stdev [Mpps]".format(item["title"])]) + history = table.get(u"history", list()) + for item in history: + header.extend( + [ + f"{item[u'title']} {hdr_param} [Mpps]", + f"{item[u'title']} Stdev [Mpps]" + ] + ) header.extend( - ["{0} {1} [Mpps]".format(table["reference"]["title"], hdr_param), - "{0} Stdev [Mpps]".format(table["reference"]["title"]), - "{0} {1} [Mpps]".format(table["compare"]["title"], hdr_param), - "{0} Stdev [Mpps]".format(table["compare"]["title"]), - "Delta [%]"]) - header_str = ",".join(header) + "\n" + [ + f"{table[u'reference'][u'title']} {hdr_param} [Mpps]", + f"{table[u'reference'][u'title']} Stdev [Mpps]", + f"{table[u'compare'][u'title']} {hdr_param} [Mpps]", + f"{table[u'compare'][u'title']} Stdev [Mpps]", + u"Delta [%]", + u"Stdev of delta [%]" + ] + ) + header_str = u";".join(header) + u"\n" except (AttributeError, KeyError) as err: - logging.error("The model is invalid, missing parameter: {0}". - format(err)) + logging.error(f"The model is invalid, missing parameter: {repr(err)}") return # Prepare data to the table: tbl_dict = dict() - for job, builds in table["reference"]["data"].items(): - topo = "2n-skx" if "2n-skx" in job else "" + for job, builds in table[u"reference"][u"data"].items(): for build in builds: - for tst_name, tst_data in data[job][str(build)].iteritems(): - tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\ - replace("-ndrpdr", "").replace("-pdrdisc", "").\ - replace("-ndrdisc", "").replace("-pdr", "").\ - replace("-ndr", "").\ - replace("1t1c", "1c").replace("2t1c", "1c").\ - replace("2t2c", "2c").replace("4t2c", "2c").\ - replace("4t4c", "4c").replace("8t4c", "4c") - if "across topologies" in table["title"].lower(): - tst_name_mod = tst_name_mod.replace("2n1l-", "") + for tst_name, tst_data in data[job][str(build)].items(): + tst_name_mod = _tpc_modify_test_name(tst_name) + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): + tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: - groups = re.search(REGEX_NIC, tst_data["parent"]) - nic = groups.group(0) if groups else "" - name = "{0}-{1}".format(nic, "-".join(tst_data["name"]. - split("-")[:-1])) - if "across testbeds" in table["title"].lower() or \ - "across topologies" in table["title"].lower(): - name = name.\ - replace("1t1c", "1c").replace("2t1c", "1c").\ - replace("2t2c", "2c").replace("4t2c", "2c").\ - replace("4t4c", "4c").replace("8t4c", "4c") - tbl_dict[tst_name_mod] = {"name": name, - "ref-data": list(), - "cmp-data": list()} - try: - # TODO: Re-work when NDRPDRDISC tests are not used - if table["include-tests"] == "MRR": - tbl_dict[tst_name_mod]["ref-data"]. \ - append(tst_data["result"]["receive-rate"].avg) - elif table["include-tests"] == "PDR": - if tst_data["type"] == "PDR": - tbl_dict[tst_name_mod]["ref-data"]. \ - append(tst_data["throughput"]["value"]) - elif tst_data["type"] == "NDRPDR": - tbl_dict[tst_name_mod]["ref-data"].append( - tst_data["throughput"]["PDR"]["LOWER"]) - elif table["include-tests"] == "NDR": - if tst_data["type"] == "NDR": - tbl_dict[tst_name_mod]["ref-data"]. \ - append(tst_data["throughput"]["value"]) - elif tst_data["type"] == "NDRPDR": - tbl_dict[tst_name_mod]["ref-data"].append( - tst_data["throughput"]["NDR"]["LOWER"]) - else: - continue - except TypeError: - pass # No data in output.xml for this test + groups = re.search(REGEX_NIC, tst_data[u"parent"]) + nic = groups.group(0) if groups else u"" + name = \ + f"{nic}-{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}" + if u"across testbeds" in table[u"title"].lower() or \ + u"across topologies" in table[u"title"].lower(): + name = _tpc_modify_displayed_test_name(name) + tbl_dict[tst_name_mod] = { + u"name": name, + u"ref-data": list(), + u"cmp-data": list() + } + _tpc_insert_data(target=tbl_dict[tst_name_mod][u"ref-data"], + src=tst_data, + include_tests=table[u"include-tests"]) + + replacement = table[u"reference"].get(u"data-replacement", None) + if replacement: + create_new_list = True + rpl_data = input_data.filter_data( + table, data=replacement, continue_on_error=True) + for job, builds in replacement.items(): + for build in builds: + for tst_name, tst_data in rpl_data[job][str(build)].items(): + tst_name_mod = _tpc_modify_test_name(tst_name) + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): + tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") + if tbl_dict.get(tst_name_mod, None) is None: + name = \ + f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}" + if u"across testbeds" in table[u"title"].lower() or \ + u"across topologies" in table[u"title"].lower(): + name = _tpc_modify_displayed_test_name(name) + tbl_dict[tst_name_mod] = { + u"name": name, + u"ref-data": list(), + u"cmp-data": list() + } + if create_new_list: + create_new_list = False + tbl_dict[tst_name_mod][u"ref-data"] = list() + + _tpc_insert_data( + target=tbl_dict[tst_name_mod][u"ref-data"], + src=tst_data, + include_tests=table[u"include-tests"] + ) - for job, builds in table["compare"]["data"].items(): + for job, builds in table[u"compare"][u"data"].items(): for build in builds: - for tst_name, tst_data in data[job][str(build)].iteritems(): - tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \ - replace("-ndrpdr", "").replace("-pdrdisc", ""). \ - replace("-ndrdisc", "").replace("-pdr", ""). \ - replace("-ndr", "").\ - replace("1t1c", "1c").replace("2t1c", "1c").\ - replace("2t2c", "2c").replace("4t2c", "2c").\ - replace("4t4c", "4c").replace("8t4c", "4c") - if "across topologies" in table["title"].lower(): - tst_name_mod = tst_name_mod.replace("2n1l-", "") + for tst_name, tst_data in data[job][str(build)].items(): + tst_name_mod = _tpc_modify_test_name(tst_name) + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): + tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: - groups = re.search(REGEX_NIC, tst_data["parent"]) - nic = groups.group(0) if groups else "" - name = "{0}-{1}".format(nic, "-".join(tst_data["name"]. - split("-")[:-1])) - if "across testbeds" in table["title"].lower() or \ - "across topologies" in table["title"].lower(): - name = name.\ - replace("1t1c", "1c").replace("2t1c", "1c").\ - replace("2t2c", "2c").replace("4t2c", "2c").\ - replace("4t4c", "4c").replace("8t4c", "4c") - tbl_dict[tst_name_mod] = {"name": name, - "ref-data": list(), - "cmp-data": list()} - try: - # TODO: Re-work when NDRPDRDISC tests are not used - if table["include-tests"] == "MRR": - tbl_dict[tst_name_mod]["cmp-data"]. \ - append(tst_data["result"]["receive-rate"].avg) - elif table["include-tests"] == "PDR": - if tst_data["type"] == "PDR": - tbl_dict[tst_name_mod]["cmp-data"]. \ - append(tst_data["throughput"]["value"]) - elif tst_data["type"] == "NDRPDR": - tbl_dict[tst_name_mod]["cmp-data"].append( - tst_data["throughput"]["PDR"]["LOWER"]) - elif table["include-tests"] == "NDR": - if tst_data["type"] == "NDR": - tbl_dict[tst_name_mod]["cmp-data"]. \ - append(tst_data["throughput"]["value"]) - elif tst_data["type"] == "NDRPDR": - tbl_dict[tst_name_mod]["cmp-data"].append( - tst_data["throughput"]["NDR"]["LOWER"]) - else: + groups = re.search(REGEX_NIC, tst_data[u"parent"]) + nic = groups.group(0) if groups else u"" + name = \ + f"{nic}-{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}" + if u"across testbeds" in table[u"title"].lower() or \ + u"across topologies" in table[u"title"].lower(): + name = _tpc_modify_displayed_test_name(name) + tbl_dict[tst_name_mod] = { + u"name": name, + u"ref-data": list(), + u"cmp-data": list() + } + _tpc_insert_data( + target=tbl_dict[tst_name_mod][u"cmp-data"], + src=tst_data, + include_tests=table[u"include-tests"] + ) + + replacement = table[u"compare"].get(u"data-replacement", None) + if replacement: + create_new_list = True + rpl_data = input_data.filter_data( + table, data=replacement, continue_on_error=True) + for job, builds in replacement.items(): + for build in builds: + for tst_name, tst_data in rpl_data[job][str(build)].items(): + tst_name_mod = _tpc_modify_test_name(tst_name) + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): + tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") + if tbl_dict.get(tst_name_mod, None) is None: + name = \ + f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}" + if u"across testbeds" in table[u"title"].lower() or \ + u"across topologies" in table[u"title"].lower(): + name = _tpc_modify_displayed_test_name(name) + tbl_dict[tst_name_mod] = { + u"name": name, + u"ref-data": list(), + u"cmp-data": list() + } + if create_new_list: + create_new_list = False + tbl_dict[tst_name_mod][u"cmp-data"] = list() + + _tpc_insert_data( + target=tbl_dict[tst_name_mod][u"cmp-data"], + src=tst_data, + include_tests=table[u"include-tests"] + ) + + for item in history: + for job, builds in item[u"data"].items(): + for build in builds: + for tst_name, tst_data in data[job][str(build)].items(): + tst_name_mod = _tpc_modify_test_name(tst_name) + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): + tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") + if tbl_dict.get(tst_name_mod, None) is None: continue - except (KeyError, TypeError): - pass - if history: - for item in history: - for job, builds in item["data"].items(): - for build in builds: - for tst_name, tst_data in data[job][str(build)].iteritems(): - tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \ - replace("-ndrpdr", "").replace("-pdrdisc", ""). \ - replace("-ndrdisc", "").replace("-pdr", ""). \ - replace("-ndr", "").\ - replace("1t1c", "1c").replace("2t1c", "1c").\ - replace("2t2c", "2c").replace("4t2c", "2c").\ - replace("4t4c", "4c").replace("8t4c", "4c") - if "across topologies" in table["title"].lower(): - tst_name_mod = tst_name_mod.replace("2n1l-", "") - if tbl_dict.get(tst_name_mod, None) is None: + if tbl_dict[tst_name_mod].get(u"history", None) is None: + tbl_dict[tst_name_mod][u"history"] = OrderedDict() + if tbl_dict[tst_name_mod][u"history"].\ + get(item[u"title"], None) is None: + tbl_dict[tst_name_mod][u"history"][item[ + u"title"]] = list() + try: + if table[u"include-tests"] == u"MRR": + res = (tst_data[u"result"][u"receive-rate"], + tst_data[u"result"][u"receive-stdev"]) + elif table[u"include-tests"] == u"PDR": + res = tst_data[u"throughput"][u"PDR"][u"LOWER"] + elif table[u"include-tests"] == u"NDR": + res = tst_data[u"throughput"][u"NDR"][u"LOWER"] + else: continue - if tbl_dict[tst_name_mod].get("history", None) is None: - tbl_dict[tst_name_mod]["history"] = OrderedDict() - if tbl_dict[tst_name_mod]["history"].get(item["title"], - None) is None: - tbl_dict[tst_name_mod]["history"][item["title"]] = \ - list() - try: - # TODO: Re-work when NDRPDRDISC tests are not used - if table["include-tests"] == "MRR": - tbl_dict[tst_name_mod]["history"][item["title" - ]].append(tst_data["result"]["receive-rate"]. - avg) - elif table["include-tests"] == "PDR": - if tst_data["type"] == "PDR": - tbl_dict[tst_name_mod]["history"][ - item["title"]].\ - append(tst_data["throughput"]["value"]) - elif tst_data["type"] == "NDRPDR": - tbl_dict[tst_name_mod]["history"][item[ - "title"]].append(tst_data["throughput"][ - "PDR"]["LOWER"]) - elif table["include-tests"] == "NDR": - if tst_data["type"] == "NDR": - tbl_dict[tst_name_mod]["history"][ - item["title"]].\ - append(tst_data["throughput"]["value"]) - elif tst_data["type"] == "NDRPDR": - tbl_dict[tst_name_mod]["history"][item[ - "title"]].append(tst_data["throughput"][ - "NDR"]["LOWER"]) - else: - continue - except (TypeError, KeyError): - pass + tbl_dict[tst_name_mod][u"history"][item[u"title"]].\ + append(res) + except (TypeError, KeyError): + pass tbl_lst = list() - footnote = False - for tst_name in tbl_dict.keys(): - item = [tbl_dict[tst_name]["name"], ] + for tst_name in tbl_dict: + item = [tbl_dict[tst_name][u"name"], ] if history: - if tbl_dict[tst_name].get("history", None) is not None: - for hist_data in tbl_dict[tst_name]["history"].values(): + if tbl_dict[tst_name].get(u"history", None) is not None: + for hist_data in tbl_dict[tst_name][u"history"].values(): if hist_data: - item.append(round(mean(hist_data) / 1000000, 2)) - item.append(round(stdev(hist_data) / 1000000, 2)) + if table[u"include-tests"] == u"MRR": + item.append(round(hist_data[0][0] / 1e6, 2)) + item.append(round(hist_data[0][1] / 1e6, 2)) + else: + item.append(round(mean(hist_data) / 1e6, 2)) + item.append(round(stdev(hist_data) / 1e6, 2)) else: - item.extend(["Not tested", "Not tested"]) + item.extend([u"Not tested", u"Not tested"]) + else: + item.extend([u"Not tested", u"Not tested"]) + data_r = tbl_dict[tst_name][u"ref-data"] + if data_r: + if table[u"include-tests"] == u"MRR": + data_r_mean = data_r[0][0] + data_r_stdev = data_r[0][1] else: - item.extend(["Not tested", "Not tested"]) - data_t = tbl_dict[tst_name]["ref-data"] - if data_t: - item.append(round(mean(data_t) / 1000000, 2)) - item.append(round(stdev(data_t) / 1000000, 2)) + data_r_mean = mean(data_r) + data_r_stdev = stdev(data_r) + item.append(round(data_r_mean / 1e6, 2)) + item.append(round(data_r_stdev / 1e6, 2)) else: - item.extend(["Not tested", "Not tested"]) - data_t = tbl_dict[tst_name]["cmp-data"] - if data_t: - item.append(round(mean(data_t) / 1000000, 2)) - item.append(round(stdev(data_t) / 1000000, 2)) + data_r_mean = None + data_r_stdev = None + item.extend([u"Not tested", u"Not tested"]) + data_c = tbl_dict[tst_name][u"cmp-data"] + if data_c: + if table[u"include-tests"] == u"MRR": + data_c_mean = data_c[0][0] + data_c_stdev = data_c[0][1] + else: + data_c_mean = mean(data_c) + data_c_stdev = stdev(data_c) + item.append(round(data_c_mean / 1e6, 2)) + item.append(round(data_c_stdev / 1e6, 2)) else: - item.extend(["Not tested", "Not tested"]) - if item[-2] == "Not tested": + data_c_mean = None + data_c_stdev = None + item.extend([u"Not tested", u"Not tested"]) + if item[-2] == u"Not tested": pass - elif item[-4] == "Not tested": - item.append("New in CSIT-1908") - elif topo == "2n-skx" and "dot1q" in tbl_dict[tst_name]["name"]: - item.append("See footnote [1]") - footnote = True - elif item[-4] != 0: - item.append(int(relative_change(float(item[-4]), float(item[-2])))) - if (len(item) == len(header)) and (item[-3] != "Not tested"): + elif item[-4] == u"Not tested": + item.append(u"New in CSIT-2001") + item.append(u"New in CSIT-2001") + elif data_r_mean is not None and data_c_mean is not None: + delta, d_stdev = relative_change_stdev( + data_r_mean, data_c_mean, data_r_stdev, data_c_stdev + ) + try: + item.append(round(delta)) + except ValueError: + item.append(delta) + try: + item.append(round(d_stdev)) + except ValueError: + item.append(d_stdev) + if rca_data: + rca_nr = rca_data.get(item[0], u"-") + item.insert(0, f"[{rca_nr}]" if rca_nr != u"-" else u"-") + if (len(item) == len(header)) and (item[-4] != u"Not tested"): tbl_lst.append(item) - # Sort the table according to the relative change - tbl_lst.sort(key=lambda rel: rel[-1], reverse=True) + tbl_lst = _tpc_sort_table(tbl_lst) # Generate csv tables: - csv_file = "{0}.csv".format(table["output-file"]) - with open(csv_file, "w") as file_handler: + csv_file = f"{table[u'output-file']}.csv" + with open(csv_file, u"wt") as file_handler: file_handler.write(header_str) for test in tbl_lst: - file_handler.write(",".join([str(item) for item in test]) + "\n") - - txt_file_name = "{0}.txt".format(table["output-file"]) - convert_csv_to_pretty_txt(csv_file, txt_file_name) - - if footnote: - with open(txt_file_name, 'a') as txt_file: - txt_file.writelines([ - "\nFootnotes:\n", - "[1] CSIT-1908 changed test methodology of dot1q tests in " - "2n-skx testbeds, dot1q encapsulation is now used on both " - "links of SUT.\n", - " Previously dot1q was used only on a single link with the " - "other link carrying untagged Ethernet frames. This change " - "results\n", - " in slightly lower throughput in CSIT-1908 for these " - "tests. See release notes." - ]) - - -def table_performance_comparison_nic(table, input_data): - """Generate the table(s) with algorithm: table_performance_comparison + file_handler.write(u";".join([str(item) for item in test]) + u"\n") + + txt_file_name = f"{table[u'output-file']}.txt" + convert_csv_to_pretty_txt(csv_file, txt_file_name, delimiter=u";") + + if rca_data: + footnote = rca_data.get(u"footnote", "") + if footnote: + with open(txt_file_name, u'a') as txt_file: + txt_file.writelines(footnote) + + # Generate html table: + _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html") + + +def table_perf_comparison_nic(table, input_data): + """Generate the table(s) with algorithm: table_perf_comparison specified in the specification file. :param table: Table to generate. @@ -461,264 +901,305 @@ def table_performance_comparison_nic(table, input_data): :type input_data: InputData """ - logging.info(" Generating the table {0} ...". - format(table.get("title", ""))) + logging.info(f" Generating the table {table.get(u'title', u'')} ...") # Transform the data - logging.info(" Creating the data set for the {0} '{1}'.". - format(table.get("type", ""), table.get("title", ""))) + logging.info( + f" Creating the data set for the {table.get(u'type', u'')} " + f"{table.get(u'title', u'')}." + ) data = input_data.filter_data(table, continue_on_error=True) # Prepare the header of the tables try: - header = ["Test case", ] + header = [u"Test case", ] - if table["include-tests"] == "MRR": - hdr_param = "Rec Rate" + rca_data = None + rca = table.get(u"rca", None) + if rca: + try: + with open(rca.get(u"data-file", ""), u"r") as rca_file: + rca_data = load(rca_file, Loader=FullLoader) + header.insert(0, rca.get(u"title", "RCA")) + except (YAMLError, IOError) as err: + logging.warning(repr(err)) + + if table[u"include-tests"] == u"MRR": + hdr_param = u"Rec Rate" else: - hdr_param = "Thput" + hdr_param = u"Thput" - history = table.get("history", None) - if history: - for item in history: - header.extend( - ["{0} {1} [Mpps]".format(item["title"], hdr_param), - "{0} Stdev [Mpps]".format(item["title"])]) + history = table.get(u"history", list()) + for item in history: + header.extend( + [ + f"{item[u'title']} {hdr_param} [Mpps]", + f"{item[u'title']} Stdev [Mpps]" + ] + ) header.extend( - ["{0} {1} [Mpps]".format(table["reference"]["title"], hdr_param), - "{0} Stdev [Mpps]".format(table["reference"]["title"]), - "{0} {1} [Mpps]".format(table["compare"]["title"], hdr_param), - "{0} Stdev [Mpps]".format(table["compare"]["title"]), - "Delta [%]"]) - header_str = ",".join(header) + "\n" + [ + f"{table[u'reference'][u'title']} {hdr_param} [Mpps]", + f"{table[u'reference'][u'title']} Stdev [Mpps]", + f"{table[u'compare'][u'title']} {hdr_param} [Mpps]", + f"{table[u'compare'][u'title']} Stdev [Mpps]", + u"Delta [%]", + u"Stdev of delta [%]" + ] + ) + header_str = u";".join(header) + u"\n" except (AttributeError, KeyError) as err: - logging.error("The model is invalid, missing parameter: {0}". - format(err)) + logging.error(f"The model is invalid, missing parameter: {repr(err)}") return # Prepare data to the table: tbl_dict = dict() - for job, builds in table["reference"]["data"].items(): - topo = "2n-skx" if "2n-skx" in job else "" + for job, builds in table[u"reference"][u"data"].items(): for build in builds: - for tst_name, tst_data in data[job][str(build)].iteritems(): - if table["reference"]["nic"] not in tst_data["tags"]: + for tst_name, tst_data in data[job][str(build)].items(): + if table[u"reference"][u"nic"] not in tst_data[u"tags"]: continue - tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\ - replace("-ndrpdr", "").replace("-pdrdisc", "").\ - replace("-ndrdisc", "").replace("-pdr", "").\ - replace("-ndr", "").\ - replace("1t1c", "1c").replace("2t1c", "1c").\ - replace("2t2c", "2c").replace("4t2c", "2c").\ - replace("4t4c", "4c").replace("8t4c", "4c") - tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod) - if "across topologies" in table["title"].lower(): - tst_name_mod = tst_name_mod.replace("2n1l-", "") + tst_name_mod = _tpc_modify_test_name(tst_name) + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): + tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: - name = "{0}".format("-".join(tst_data["name"]. - split("-")[:-1])) - if "across testbeds" in table["title"].lower() or \ - "across topologies" in table["title"].lower(): - name = name.\ - replace("1t1c", "1c").replace("2t1c", "1c").\ - replace("2t2c", "2c").replace("4t2c", "2c").\ - replace("4t4c", "4c").replace("8t4c", "4c") - tbl_dict[tst_name_mod] = {"name": name, - "ref-data": list(), - "cmp-data": list()} - try: - # TODO: Re-work when NDRPDRDISC tests are not used - if table["include-tests"] == "MRR": - tbl_dict[tst_name_mod]["ref-data"]. \ - append(tst_data["result"]["receive-rate"].avg) - elif table["include-tests"] == "PDR": - if tst_data["type"] == "PDR": - tbl_dict[tst_name_mod]["ref-data"]. \ - append(tst_data["throughput"]["value"]) - elif tst_data["type"] == "NDRPDR": - tbl_dict[tst_name_mod]["ref-data"].append( - tst_data["throughput"]["PDR"]["LOWER"]) - elif table["include-tests"] == "NDR": - if tst_data["type"] == "NDR": - tbl_dict[tst_name_mod]["ref-data"]. \ - append(tst_data["throughput"]["value"]) - elif tst_data["type"] == "NDRPDR": - tbl_dict[tst_name_mod]["ref-data"].append( - tst_data["throughput"]["NDR"]["LOWER"]) - else: + name = f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}" + if u"across testbeds" in table[u"title"].lower() or \ + u"across topologies" in table[u"title"].lower(): + name = _tpc_modify_displayed_test_name(name) + tbl_dict[tst_name_mod] = { + u"name": name, + u"ref-data": list(), + u"cmp-data": list() + } + _tpc_insert_data( + target=tbl_dict[tst_name_mod][u"ref-data"], + src=tst_data, + include_tests=table[u"include-tests"] + ) + + replacement = table[u"reference"].get(u"data-replacement", None) + if replacement: + create_new_list = True + rpl_data = input_data.filter_data( + table, data=replacement, continue_on_error=True) + for job, builds in replacement.items(): + for build in builds: + for tst_name, tst_data in rpl_data[job][str(build)].items(): + if table[u"reference"][u"nic"] not in tst_data[u"tags"]: continue - except TypeError: - pass # No data in output.xml for this test + tst_name_mod = _tpc_modify_test_name(tst_name) + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): + tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") + if tbl_dict.get(tst_name_mod, None) is None: + name = \ + f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}" + if u"across testbeds" in table[u"title"].lower() or \ + u"across topologies" in table[u"title"].lower(): + name = _tpc_modify_displayed_test_name(name) + tbl_dict[tst_name_mod] = { + u"name": name, + u"ref-data": list(), + u"cmp-data": list() + } + if create_new_list: + create_new_list = False + tbl_dict[tst_name_mod][u"ref-data"] = list() + + _tpc_insert_data( + target=tbl_dict[tst_name_mod][u"ref-data"], + src=tst_data, + include_tests=table[u"include-tests"] + ) - for job, builds in table["compare"]["data"].items(): + for job, builds in table[u"compare"][u"data"].items(): for build in builds: - for tst_name, tst_data in data[job][str(build)].iteritems(): - if table["compare"]["nic"] not in tst_data["tags"]: + for tst_name, tst_data in data[job][str(build)].items(): + if table[u"compare"][u"nic"] not in tst_data[u"tags"]: continue - tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \ - replace("-ndrpdr", "").replace("-pdrdisc", ""). \ - replace("-ndrdisc", "").replace("-pdr", ""). \ - replace("-ndr", "").\ - replace("1t1c", "1c").replace("2t1c", "1c").\ - replace("2t2c", "2c").replace("4t2c", "2c").\ - replace("4t4c", "4c").replace("8t4c", "4c") - tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod) - if "across topologies" in table["title"].lower(): - tst_name_mod = tst_name_mod.replace("2n1l-", "") + tst_name_mod = _tpc_modify_test_name(tst_name) + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): + tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: - name = "{0}".format("-".join(tst_data["name"]. - split("-")[:-1])) - if "across testbeds" in table["title"].lower() or \ - "across topologies" in table["title"].lower(): - name = name.\ - replace("1t1c", "1c").replace("2t1c", "1c").\ - replace("2t2c", "2c").replace("4t2c", "2c").\ - replace("4t4c", "4c").replace("8t4c", "4c") - tbl_dict[tst_name_mod] = {"name": name, - "ref-data": list(), - "cmp-data": list()} - try: - # TODO: Re-work when NDRPDRDISC tests are not used - if table["include-tests"] == "MRR": - tbl_dict[tst_name_mod]["cmp-data"]. \ - append(tst_data["result"]["receive-rate"].avg) - elif table["include-tests"] == "PDR": - if tst_data["type"] == "PDR": - tbl_dict[tst_name_mod]["cmp-data"]. \ - append(tst_data["throughput"]["value"]) - elif tst_data["type"] == "NDRPDR": - tbl_dict[tst_name_mod]["cmp-data"].append( - tst_data["throughput"]["PDR"]["LOWER"]) - elif table["include-tests"] == "NDR": - if tst_data["type"] == "NDR": - tbl_dict[tst_name_mod]["cmp-data"]. \ - append(tst_data["throughput"]["value"]) - elif tst_data["type"] == "NDRPDR": - tbl_dict[tst_name_mod]["cmp-data"].append( - tst_data["throughput"]["NDR"]["LOWER"]) - else: + name = f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}" + if u"across testbeds" in table[u"title"].lower() or \ + u"across topologies" in table[u"title"].lower(): + name = _tpc_modify_displayed_test_name(name) + tbl_dict[tst_name_mod] = { + u"name": name, + u"ref-data": list(), + u"cmp-data": list() + } + _tpc_insert_data( + target=tbl_dict[tst_name_mod][u"cmp-data"], + src=tst_data, + include_tests=table[u"include-tests"] + ) + + replacement = table[u"compare"].get(u"data-replacement", None) + if replacement: + create_new_list = True + rpl_data = input_data.filter_data( + table, data=replacement, continue_on_error=True) + for job, builds in replacement.items(): + for build in builds: + for tst_name, tst_data in rpl_data[job][str(build)].items(): + if table[u"compare"][u"nic"] not in tst_data[u"tags"]: continue - except (KeyError, TypeError): - pass - - if history: - for item in history: - for job, builds in item["data"].items(): - for build in builds: - for tst_name, tst_data in data[job][str(build)].iteritems(): - if item["nic"] not in tst_data["tags"]: - continue - tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \ - replace("-ndrpdr", "").replace("-pdrdisc", ""). \ - replace("-ndrdisc", "").replace("-pdr", ""). \ - replace("-ndr", "").\ - replace("1t1c", "1c").replace("2t1c", "1c").\ - replace("2t2c", "2c").replace("4t2c", "2c").\ - replace("4t4c", "4c").replace("8t4c", "4c") - tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod) - if "across topologies" in table["title"].lower(): - tst_name_mod = tst_name_mod.replace("2n1l-", "") - if tbl_dict.get(tst_name_mod, None) is None: + tst_name_mod = _tpc_modify_test_name(tst_name) + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): + tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") + if tbl_dict.get(tst_name_mod, None) is None: + name = \ + f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}" + if u"across testbeds" in table[u"title"].lower() or \ + u"across topologies" in table[u"title"].lower(): + name = _tpc_modify_displayed_test_name(name) + tbl_dict[tst_name_mod] = { + u"name": name, + u"ref-data": list(), + u"cmp-data": list() + } + if create_new_list: + create_new_list = False + tbl_dict[tst_name_mod][u"cmp-data"] = list() + + _tpc_insert_data( + target=tbl_dict[tst_name_mod][u"cmp-data"], + src=tst_data, + include_tests=table[u"include-tests"] + ) + + for item in history: + for job, builds in item[u"data"].items(): + for build in builds: + for tst_name, tst_data in data[job][str(build)].items(): + if item[u"nic"] not in tst_data[u"tags"]: + continue + tst_name_mod = _tpc_modify_test_name(tst_name) + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): + tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") + if tbl_dict.get(tst_name_mod, None) is None: + continue + if tbl_dict[tst_name_mod].get(u"history", None) is None: + tbl_dict[tst_name_mod][u"history"] = OrderedDict() + if tbl_dict[tst_name_mod][u"history"].\ + get(item[u"title"], None) is None: + tbl_dict[tst_name_mod][u"history"][item[ + u"title"]] = list() + try: + if table[u"include-tests"] == u"MRR": + res = (tst_data[u"result"][u"receive-rate"], + tst_data[u"result"][u"receive-stdev"]) + elif table[u"include-tests"] == u"PDR": + res = tst_data[u"throughput"][u"PDR"][u"LOWER"] + elif table[u"include-tests"] == u"NDR": + res = tst_data[u"throughput"][u"NDR"][u"LOWER"] + else: continue - if tbl_dict[tst_name_mod].get("history", None) is None: - tbl_dict[tst_name_mod]["history"] = OrderedDict() - if tbl_dict[tst_name_mod]["history"].get(item["title"], - None) is None: - tbl_dict[tst_name_mod]["history"][item["title"]] = \ - list() - try: - # TODO: Re-work when NDRPDRDISC tests are not used - if table["include-tests"] == "MRR": - tbl_dict[tst_name_mod]["history"][item["title" - ]].append(tst_data["result"]["receive-rate"]. - avg) - elif table["include-tests"] == "PDR": - if tst_data["type"] == "PDR": - tbl_dict[tst_name_mod]["history"][ - item["title"]].\ - append(tst_data["throughput"]["value"]) - elif tst_data["type"] == "NDRPDR": - tbl_dict[tst_name_mod]["history"][item[ - "title"]].append(tst_data["throughput"][ - "PDR"]["LOWER"]) - elif table["include-tests"] == "NDR": - if tst_data["type"] == "NDR": - tbl_dict[tst_name_mod]["history"][ - item["title"]].\ - append(tst_data["throughput"]["value"]) - elif tst_data["type"] == "NDRPDR": - tbl_dict[tst_name_mod]["history"][item[ - "title"]].append(tst_data["throughput"][ - "NDR"]["LOWER"]) - else: - continue - except (TypeError, KeyError): - pass + tbl_dict[tst_name_mod][u"history"][item[u"title"]].\ + append(res) + except (TypeError, KeyError): + pass tbl_lst = list() - footnote = False - for tst_name in tbl_dict.keys(): - item = [tbl_dict[tst_name]["name"], ] + for tst_name in tbl_dict: + item = [tbl_dict[tst_name][u"name"], ] if history: - if tbl_dict[tst_name].get("history", None) is not None: - for hist_data in tbl_dict[tst_name]["history"].values(): + if tbl_dict[tst_name].get(u"history", None) is not None: + for hist_data in tbl_dict[tst_name][u"history"].values(): if hist_data: - item.append(round(mean(hist_data) / 1000000, 2)) - item.append(round(stdev(hist_data) / 1000000, 2)) + if table[u"include-tests"] == u"MRR": + item.append(round(hist_data[0][0] / 1e6, 2)) + item.append(round(hist_data[0][1] / 1e6, 2)) + else: + item.append(round(mean(hist_data) / 1e6, 2)) + item.append(round(stdev(hist_data) / 1e6, 2)) else: - item.extend(["Not tested", "Not tested"]) + item.extend([u"Not tested", u"Not tested"]) else: - item.extend(["Not tested", "Not tested"]) - data_t = tbl_dict[tst_name]["ref-data"] - if data_t: - item.append(round(mean(data_t) / 1000000, 2)) - item.append(round(stdev(data_t) / 1000000, 2)) + item.extend([u"Not tested", u"Not tested"]) + data_r = tbl_dict[tst_name][u"ref-data"] + if data_r: + if table[u"include-tests"] == u"MRR": + data_r_mean = data_r[0][0] + data_r_stdev = data_r[0][1] + else: + data_r_mean = mean(data_r) + data_r_stdev = stdev(data_r) + item.append(round(data_r_mean / 1e6, 2)) + item.append(round(data_r_stdev / 1e6, 2)) else: - item.extend(["Not tested", "Not tested"]) - data_t = tbl_dict[tst_name]["cmp-data"] - if data_t: - item.append(round(mean(data_t) / 1000000, 2)) - item.append(round(stdev(data_t) / 1000000, 2)) + data_r_mean = None + data_r_stdev = None + item.extend([u"Not tested", u"Not tested"]) + data_c = tbl_dict[tst_name][u"cmp-data"] + if data_c: + if table[u"include-tests"] == u"MRR": + data_c_mean = data_c[0][0] + data_c_stdev = data_c[0][1] + else: + data_c_mean = mean(data_c) + data_c_stdev = stdev(data_c) + item.append(round(data_c_mean / 1e6, 2)) + item.append(round(data_c_stdev / 1e6, 2)) else: - item.extend(["Not tested", "Not tested"]) - if item[-2] == "Not tested": + data_c_mean = None + data_c_stdev = None + item.extend([u"Not tested", u"Not tested"]) + if item[-2] == u"Not tested": pass - elif item[-4] == "Not tested": - item.append("New in CSIT-1908") - elif topo == "2n-skx" and "dot1q" in tbl_dict[tst_name]["name"]: - item.append("See footnote [1]") - footnote = True - elif item[-4] != 0: - item.append(int(relative_change(float(item[-4]), float(item[-2])))) - if (len(item) == len(header)) and (item[-3] != "Not tested"): + elif item[-4] == u"Not tested": + item.append(u"New in CSIT-2001") + item.append(u"New in CSIT-2001") + elif data_r_mean is not None and data_c_mean is not None: + delta, d_stdev = relative_change_stdev( + data_r_mean, data_c_mean, data_r_stdev, data_c_stdev + ) + try: + item.append(round(delta)) + except ValueError: + item.append(delta) + try: + item.append(round(d_stdev)) + except ValueError: + item.append(d_stdev) + if rca_data: + rca_nr = rca_data.get(item[0], u"-") + item.insert(0, f"[{rca_nr}]" if rca_nr != u"-" else u"-") + if (len(item) == len(header)) and (item[-4] != u"Not tested"): tbl_lst.append(item) - # Sort the table according to the relative change - tbl_lst.sort(key=lambda rel: rel[-1], reverse=True) + tbl_lst = _tpc_sort_table(tbl_lst) # Generate csv tables: - csv_file = "{0}.csv".format(table["output-file"]) - with open(csv_file, "w") as file_handler: + csv_file = f"{table[u'output-file']}.csv" + with open(csv_file, u"wt") as file_handler: file_handler.write(header_str) for test in tbl_lst: - file_handler.write(",".join([str(item) for item in test]) + "\n") - - txt_file_name = "{0}.txt".format(table["output-file"]) - convert_csv_to_pretty_txt(csv_file, txt_file_name) - - if footnote: - with open(txt_file_name, 'a') as txt_file: - txt_file.writelines([ - "\nFootnotes:\n", - "[1] CSIT-1908 changed test methodology of dot1q tests in " - "2n-skx testbeds, dot1q encapsulation is now used on both " - "links of SUT.\n", - " Previously dot1q was used only on a single link with the " - "other link carrying untagged Ethernet frames. This change " - "results\n", - " in slightly lower throughput in CSIT-1908 for these " - "tests. See release notes." - ]) + file_handler.write(u";".join([str(item) for item in test]) + u"\n") + + txt_file_name = f"{table[u'output-file']}.txt" + convert_csv_to_pretty_txt(csv_file, txt_file_name, delimiter=u";") + + if rca_data: + footnote = rca_data.get(u"footnote", "") + if footnote: + with open(txt_file_name, u'a') as txt_file: + txt_file.writelines(footnote) + + # Generate html table: + _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html") def table_nics_comparison(table, input_data): @@ -731,104 +1212,132 @@ def table_nics_comparison(table, input_data): :type input_data: InputData """ - logging.info(" Generating the table {0} ...". - format(table.get("title", ""))) + logging.info(f" Generating the table {table.get(u'title', u'')} ...") # Transform the data - logging.info(" Creating the data set for the {0} '{1}'.". - format(table.get("type", ""), table.get("title", ""))) + logging.info( + f" Creating the data set for the {table.get(u'type', u'')} " + f"{table.get(u'title', u'')}." + ) data = input_data.filter_data(table, continue_on_error=True) # Prepare the header of the tables try: - header = ["Test case", ] + header = [u"Test case", ] - if table["include-tests"] == "MRR": - hdr_param = "Rec Rate" + if table[u"include-tests"] == u"MRR": + hdr_param = u"Rec Rate" else: - hdr_param = "Thput" + hdr_param = u"Thput" header.extend( - ["{0} {1} [Mpps]".format(table["reference"]["title"], hdr_param), - "{0} Stdev [Mpps]".format(table["reference"]["title"]), - "{0} {1} [Mpps]".format(table["compare"]["title"], hdr_param), - "{0} Stdev [Mpps]".format(table["compare"]["title"]), - "Delta [%]"]) - header_str = ",".join(header) + "\n" + [ + f"{table[u'reference'][u'title']} {hdr_param} [Mpps]", + f"{table[u'reference'][u'title']} Stdev [Mpps]", + f"{table[u'compare'][u'title']} {hdr_param} [Mpps]", + f"{table[u'compare'][u'title']} Stdev [Mpps]", + u"Delta [%]", + u"Stdev of delta [%]" + ] + ) + except (AttributeError, KeyError) as err: - logging.error("The model is invalid, missing parameter: {0}". - format(err)) + logging.error(f"The model is invalid, missing parameter: {repr(err)}") return # Prepare data to the table: tbl_dict = dict() - for job, builds in table["data"].items(): + for job, builds in table[u"data"].items(): for build in builds: - for tst_name, tst_data in data[job][str(build)].iteritems(): - tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\ - replace("-ndrpdr", "").replace("-pdrdisc", "").\ - replace("-ndrdisc", "").replace("-pdr", "").\ - replace("-ndr", "").\ - replace("1t1c", "1c").replace("2t1c", "1c").\ - replace("2t2c", "2c").replace("4t2c", "2c").\ - replace("4t4c", "4c").replace("8t4c", "4c") - tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod) + for tst_name, tst_data in data[job][str(build)].items(): + tst_name_mod = _tpc_modify_test_name(tst_name) if tbl_dict.get(tst_name_mod, None) is None: - name = "-".join(tst_data["name"].split("-")[:-1]) - tbl_dict[tst_name_mod] = {"name": name, - "ref-data": list(), - "cmp-data": list()} + name = u"-".join(tst_data[u"name"].split(u"-")[:-1]) + tbl_dict[tst_name_mod] = { + u"name": name, + u"ref-data": list(), + u"cmp-data": list() + } try: - if table["include-tests"] == "MRR": - result = tst_data["result"]["receive-rate"].avg - elif table["include-tests"] == "PDR": - result = tst_data["throughput"]["PDR"]["LOWER"] - elif table["include-tests"] == "NDR": - result = tst_data["throughput"]["NDR"]["LOWER"] + if table[u"include-tests"] == u"MRR": + result = (tst_data[u"result"][u"receive-rate"], + tst_data[u"result"][u"receive-stdev"]) + elif table[u"include-tests"] == u"PDR": + result = tst_data[u"throughput"][u"PDR"][u"LOWER"] + elif table[u"include-tests"] == u"NDR": + result = tst_data[u"throughput"][u"NDR"][u"LOWER"] else: - result = None + continue - if result: - if table["reference"]["nic"] in tst_data["tags"]: - tbl_dict[tst_name_mod]["ref-data"].append(result) - elif table["compare"]["nic"] in tst_data["tags"]: - tbl_dict[tst_name_mod]["cmp-data"].append(result) + if result and \ + table[u"reference"][u"nic"] in tst_data[u"tags"]: + tbl_dict[tst_name_mod][u"ref-data"].append(result) + elif result and \ + table[u"compare"][u"nic"] in tst_data[u"tags"]: + tbl_dict[tst_name_mod][u"cmp-data"].append(result) except (TypeError, KeyError) as err: - logging.debug("No data for {0}".format(tst_name)) - logging.debug(repr(err)) + logging.debug(f"No data for {tst_name}\n{repr(err)}") # No data in output.xml for this test tbl_lst = list() - for tst_name in tbl_dict.keys(): - item = [tbl_dict[tst_name]["name"], ] - data_t = tbl_dict[tst_name]["ref-data"] - if data_t: - item.append(round(mean(data_t) / 1000000, 2)) - item.append(round(stdev(data_t) / 1000000, 2)) + for tst_name in tbl_dict: + item = [tbl_dict[tst_name][u"name"], ] + data_r = tbl_dict[tst_name][u"ref-data"] + if data_r: + if table[u"include-tests"] == u"MRR": + data_r_mean = data_r[0][0] + data_r_stdev = data_r[0][1] + else: + data_r_mean = mean(data_r) + data_r_stdev = stdev(data_r) + item.append(round(data_r_mean / 1e6, 2)) + item.append(round(data_r_stdev / 1e6, 2)) else: + data_r_mean = None + data_r_stdev = None item.extend([None, None]) - data_t = tbl_dict[tst_name]["cmp-data"] - if data_t: - item.append(round(mean(data_t) / 1000000, 2)) - item.append(round(stdev(data_t) / 1000000, 2)) + data_c = tbl_dict[tst_name][u"cmp-data"] + if data_c: + if table[u"include-tests"] == u"MRR": + data_c_mean = data_c[0][0] + data_c_stdev = data_c[0][1] + else: + data_c_mean = mean(data_c) + data_c_stdev = stdev(data_c) + item.append(round(data_c_mean / 1e6, 2)) + item.append(round(data_c_stdev / 1e6, 2)) else: + data_c_mean = None + data_c_stdev = None item.extend([None, None]) - if item[-4] is not None and item[-2] is not None and item[-4] != 0: - item.append(int(relative_change(float(item[-4]), float(item[-2])))) - if len(item) == len(header): + if data_r_mean is not None and data_c_mean is not None: + delta, d_stdev = relative_change_stdev( + data_r_mean, data_c_mean, data_r_stdev, data_c_stdev + ) + try: + item.append(round(delta)) + except ValueError: + item.append(delta) + try: + item.append(round(d_stdev)) + except ValueError: + item.append(d_stdev) tbl_lst.append(item) # Sort the table according to the relative change tbl_lst.sort(key=lambda rel: rel[-1], reverse=True) # Generate csv tables: - csv_file = "{0}.csv".format(table["output-file"]) - with open(csv_file, "w") as file_handler: - file_handler.write(header_str) + with open(f"{table[u'output-file']}.csv", u"wt") as file_handler: + file_handler.write(u",".join(header) + u"\n") for test in tbl_lst: - file_handler.write(",".join([str(item) for item in test]) + "\n") + file_handler.write(u",".join([str(item) for item in test]) + u"\n") + + convert_csv_to_pretty_txt(f"{table[u'output-file']}.csv", + f"{table[u'output-file']}.txt") - convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"])) + # Generate html table: + _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html") def table_soak_vs_ndr(table, input_data): @@ -841,122 +1350,148 @@ def table_soak_vs_ndr(table, input_data): :type input_data: InputData """ - logging.info(" Generating the table {0} ...". - format(table.get("title", ""))) + logging.info(f" Generating the table {table.get(u'title', u'')} ...") # Transform the data - logging.info(" Creating the data set for the {0} '{1}'.". - format(table.get("type", ""), table.get("title", ""))) + logging.info( + f" Creating the data set for the {table.get(u'type', u'')} " + f"{table.get(u'title', u'')}." + ) data = input_data.filter_data(table, continue_on_error=True) # Prepare the header of the table try: header = [ - "Test case", - "{0} Thput [Mpps]".format(table["reference"]["title"]), - "{0} Stdev [Mpps]".format(table["reference"]["title"]), - "{0} Thput [Mpps]".format(table["compare"]["title"]), - "{0} Stdev [Mpps]".format(table["compare"]["title"]), - "Delta [%]", "Stdev of delta [%]"] - header_str = ",".join(header) + "\n" + u"Test case", + f"{table[u'reference'][u'title']} Thput [Mpps]", + f"{table[u'reference'][u'title']} Stdev [Mpps]", + f"{table[u'compare'][u'title']} Thput [Mpps]", + f"{table[u'compare'][u'title']} Stdev [Mpps]", + u"Delta [%]", + u"Stdev of delta [%]" + ] + header_str = u",".join(header) + u"\n" except (AttributeError, KeyError) as err: - logging.error("The model is invalid, missing parameter: {0}". - format(err)) + logging.error(f"The model is invalid, missing parameter: {repr(err)}") return # Create a list of available SOAK test results: tbl_dict = dict() - for job, builds in table["compare"]["data"].items(): + for job, builds in table[u"compare"][u"data"].items(): for build in builds: - for tst_name, tst_data in data[job][str(build)].iteritems(): - if tst_data["type"] == "SOAK": - tst_name_mod = tst_name.replace("-soak", "") + for tst_name, tst_data in data[job][str(build)].items(): + if tst_data[u"type"] == u"SOAK": + tst_name_mod = tst_name.replace(u"-soak", u"") if tbl_dict.get(tst_name_mod, None) is None: - groups = re.search(REGEX_NIC, tst_data["parent"]) - nic = groups.group(0) if groups else "" - name = "{0}-{1}".format(nic, "-".join(tst_data["name"]. - split("-")[:-1])) + groups = re.search(REGEX_NIC, tst_data[u"parent"]) + nic = groups.group(0) if groups else u"" + name = ( + f"{nic}-" + f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}" + ) tbl_dict[tst_name_mod] = { - "name": name, - "ref-data": list(), - "cmp-data": list() + u"name": name, + u"ref-data": list(), + u"cmp-data": list() } try: - tbl_dict[tst_name_mod]["cmp-data"].append( - tst_data["throughput"]["LOWER"]) + tbl_dict[tst_name_mod][u"cmp-data"].append( + tst_data[u"throughput"][u"LOWER"]) except (KeyError, TypeError): pass tests_lst = tbl_dict.keys() # Add corresponding NDR test results: - for job, builds in table["reference"]["data"].items(): + for job, builds in table[u"reference"][u"data"].items(): for build in builds: - for tst_name, tst_data in data[job][str(build)].iteritems(): - tst_name_mod = tst_name.replace("-ndrpdr", "").\ - replace("-mrr", "") - if tst_name_mod in tests_lst: - try: - if tst_data["type"] in ("NDRPDR", "MRR", "BMRR"): - if table["include-tests"] == "MRR": - result = tst_data["result"]["receive-rate"].avg - elif table["include-tests"] == "PDR": - result = tst_data["throughput"]["PDR"]["LOWER"] - elif table["include-tests"] == "NDR": - result = tst_data["throughput"]["NDR"]["LOWER"] - else: - result = None - if result is not None: - tbl_dict[tst_name_mod]["ref-data"].append( - result) - except (KeyError, TypeError): + for tst_name, tst_data in data[job][str(build)].items(): + tst_name_mod = tst_name.replace(u"-ndrpdr", u"").\ + replace(u"-mrr", u"") + if tst_name_mod not in tests_lst: + continue + try: + if tst_data[u"type"] not in (u"NDRPDR", u"MRR", u"BMRR"): continue + if table[u"include-tests"] == u"MRR": + result = (tst_data[u"result"][u"receive-rate"], + tst_data[u"result"][u"receive-stdev"]) + elif table[u"include-tests"] == u"PDR": + result = \ + tst_data[u"throughput"][u"PDR"][u"LOWER"] + elif table[u"include-tests"] == u"NDR": + result = \ + tst_data[u"throughput"][u"NDR"][u"LOWER"] + else: + result = None + if result is not None: + tbl_dict[tst_name_mod][u"ref-data"].append( + result) + except (KeyError, TypeError): + continue tbl_lst = list() - for tst_name in tbl_dict.keys(): - item = [tbl_dict[tst_name]["name"], ] - data_r = tbl_dict[tst_name]["ref-data"] + for tst_name in tbl_dict: + item = [tbl_dict[tst_name][u"name"], ] + data_r = tbl_dict[tst_name][u"ref-data"] if data_r: - data_r_mean = mean(data_r) - item.append(round(data_r_mean / 1000000, 2)) - data_r_stdev = stdev(data_r) - item.append(round(data_r_stdev / 1000000, 2)) + if table[u"include-tests"] == u"MRR": + data_r_mean = data_r[0][0] + data_r_stdev = data_r[0][1] + else: + data_r_mean = mean(data_r) + data_r_stdev = stdev(data_r) + item.append(round(data_r_mean / 1e6, 2)) + item.append(round(data_r_stdev / 1e6, 2)) else: data_r_mean = None data_r_stdev = None item.extend([None, None]) - data_c = tbl_dict[tst_name]["cmp-data"] + data_c = tbl_dict[tst_name][u"cmp-data"] if data_c: - data_c_mean = mean(data_c) - item.append(round(data_c_mean / 1000000, 2)) - data_c_stdev = stdev(data_c) - item.append(round(data_c_stdev / 1000000, 2)) + if table[u"include-tests"] == u"MRR": + data_c_mean = data_c[0][0] + data_c_stdev = data_c[0][1] + else: + data_c_mean = mean(data_c) + data_c_stdev = stdev(data_c) + item.append(round(data_c_mean / 1e6, 2)) + item.append(round(data_c_stdev / 1e6, 2)) else: data_c_mean = None data_c_stdev = None item.extend([None, None]) - if data_r_mean and data_c_mean: + if data_r_mean is not None and data_c_mean is not None: delta, d_stdev = relative_change_stdev( data_r_mean, data_c_mean, data_r_stdev, data_c_stdev) - item.append(round(delta, 2)) - item.append(round(d_stdev, 2)) + try: + item.append(round(delta)) + except ValueError: + item.append(delta) + try: + item.append(round(d_stdev)) + except ValueError: + item.append(d_stdev) tbl_lst.append(item) # Sort the table according to the relative change tbl_lst.sort(key=lambda rel: rel[-1], reverse=True) # Generate csv tables: - csv_file = "{0}.csv".format(table["output-file"]) - with open(csv_file, "w") as file_handler: + csv_file = f"{table[u'output-file']}.csv" + with open(csv_file, u"wt") as file_handler: file_handler.write(header_str) for test in tbl_lst: - file_handler.write(",".join([str(item) for item in test]) + "\n") + file_handler.write(u",".join([str(item) for item in test]) + u"\n") + + convert_csv_to_pretty_txt(csv_file, f"{table[u'output-file']}.txt") - convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"])) + # Generate html table: + _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html") -def table_performance_trending_dashboard(table, input_data): +def table_perf_trending_dash(table, input_data): """Generate the table(s) with algorithm: - table_performance_trending_dashboard + table_perf_trending_dash specified in the specification file. :param table: Table to generate. @@ -965,55 +1500,58 @@ def table_performance_trending_dashboard(table, input_data): :type input_data: InputData """ - logging.info(" Generating the table {0} ...". - format(table.get("title", ""))) + logging.info(f" Generating the table {table.get(u'title', u'')} ...") # Transform the data - logging.info(" Creating the data set for the {0} '{1}'.". - format(table.get("type", ""), table.get("title", ""))) + logging.info( + f" Creating the data set for the {table.get(u'type', u'')} " + f"{table.get(u'title', u'')}." + ) data = input_data.filter_data(table, continue_on_error=True) # Prepare the header of the tables - header = ["Test Case", - "Trend [Mpps]", - "Short-Term Change [%]", - "Long-Term Change [%]", - "Regressions [#]", - "Progressions [#]" - ] - header_str = ",".join(header) + "\n" + header = [ + u"Test Case", + u"Trend [Mpps]", + u"Short-Term Change [%]", + u"Long-Term Change [%]", + u"Regressions [#]", + u"Progressions [#]" + ] + header_str = u",".join(header) + u"\n" # Prepare data to the table: tbl_dict = dict() - for job, builds in table["data"].items(): + for job, builds in table[u"data"].items(): for build in builds: - for tst_name, tst_data in data[job][str(build)].iteritems(): - if tst_name.lower() in table.get("ignore-list", list()): + for tst_name, tst_data in data[job][str(build)].items(): + if tst_name.lower() in table.get(u"ignore-list", list()): continue if tbl_dict.get(tst_name, None) is None: - groups = re.search(REGEX_NIC, tst_data["parent"]) + groups = re.search(REGEX_NIC, tst_data[u"parent"]) if not groups: continue nic = groups.group(0) tbl_dict[tst_name] = { - "name": "{0}-{1}".format(nic, tst_data["name"]), - "data": OrderedDict()} + u"name": f"{nic}-{tst_data[u'name']}", + u"data": OrderedDict() + } try: - tbl_dict[tst_name]["data"][str(build)] = \ - tst_data["result"]["receive-rate"] + tbl_dict[tst_name][u"data"][str(build)] = \ + tst_data[u"result"][u"receive-rate"] except (TypeError, KeyError): pass # No data in output.xml for this test tbl_lst = list() - for tst_name in tbl_dict.keys(): - data_t = tbl_dict[tst_name]["data"] + for tst_name in tbl_dict: + data_t = tbl_dict[tst_name][u"data"] if len(data_t) < 2: continue classification_lst, avgs = classify_anomalies(data_t) - win_size = min(len(data_t), table["window"]) - long_win_size = min(len(data_t), table["long-trend-window"]) + win_size = min(len(data_t), table[u"window"]) + long_win_size = min(len(data_t), table[u"long-trend-window"]) try: max_long_avg = max( @@ -1039,48 +1577,44 @@ def table_performance_trending_dashboard(table, input_data): if classification_lst: if isnan(rel_change_last) and isnan(rel_change_long): continue - if (isnan(last_avg) or - isnan(rel_change_last) or - isnan(rel_change_long)): + if isnan(last_avg) or isnan(rel_change_last) or \ + isnan(rel_change_long): continue tbl_lst.append( - [tbl_dict[tst_name]["name"], + [tbl_dict[tst_name][u"name"], round(last_avg / 1000000, 2), rel_change_last, rel_change_long, - classification_lst[-win_size:].count("regression"), - classification_lst[-win_size:].count("progression")]) + classification_lst[-win_size:].count(u"regression"), + classification_lst[-win_size:].count(u"progression")]) tbl_lst.sort(key=lambda rel: rel[0]) tbl_sorted = list() - for nrr in range(table["window"], -1, -1): + for nrr in range(table[u"window"], -1, -1): tbl_reg = [item for item in tbl_lst if item[4] == nrr] - for nrp in range(table["window"], -1, -1): + for nrp in range(table[u"window"], -1, -1): tbl_out = [item for item in tbl_reg if item[5] == nrp] tbl_out.sort(key=lambda rel: rel[2]) tbl_sorted.extend(tbl_out) - file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"]) + file_name = f"{table[u'output-file']}{table[u'output-file-ext']}" - logging.info(" Writing file: '{0}'".format(file_name)) - with open(file_name, "w") as file_handler: + logging.info(f" Writing file: {file_name}") + with open(file_name, u"wt") as file_handler: file_handler.write(header_str) for test in tbl_sorted: - file_handler.write(",".join([str(item) for item in test]) + '\n') + file_handler.write(u",".join([str(item) for item in test]) + u'\n') - txt_file_name = "{0}.txt".format(table["output-file"]) - logging.info(" Writing file: '{0}'".format(txt_file_name)) - convert_csv_to_pretty_txt(file_name, txt_file_name) + logging.info(f" Writing file: {table[u'output-file']}.txt") + convert_csv_to_pretty_txt(file_name, f"{table[u'output-file']}.txt") -def _generate_url(base, testbed, test_name): +def _generate_url(testbed, test_name): """Generate URL to a trending plot from the name of the test case. - :param base: The base part of URL common to all test cases. :param testbed: The testbed used for testing. :param test_name: The name of the test case. - :type base: str :type testbed: str :type test_name: str :returns: The URL to the plot with the trending data for the given test @@ -1088,146 +1622,145 @@ def _generate_url(base, testbed, test_name): :rtype str """ - url = base - file_name = "" - anchor = ".html#" - feature = "" - - if "lbdpdk" in test_name or "lbvpp" in test_name: - file_name = "link_bonding" - - elif "114b" in test_name and "vhost" in test_name: - file_name = "vts" + if u"x520" in test_name: + nic = u"x520" + elif u"x710" in test_name: + nic = u"x710" + elif u"xl710" in test_name: + nic = u"xl710" + elif u"xxv710" in test_name: + nic = u"xxv710" + elif u"vic1227" in test_name: + nic = u"vic1227" + elif u"vic1385" in test_name: + nic = u"vic1385" + elif u"x553" in test_name: + nic = u"x553" + elif u"cx556" in test_name or u"cx556a" in test_name: + nic = u"cx556a" + else: + nic = u"" + + if u"64b" in test_name: + frame_size = u"64b" + elif u"78b" in test_name: + frame_size = u"78b" + elif u"imix" in test_name: + frame_size = u"imix" + elif u"9000b" in test_name: + frame_size = u"9000b" + elif u"1518b" in test_name: + frame_size = u"1518b" + elif u"114b" in test_name: + frame_size = u"114b" + else: + frame_size = u"" + + if u"1t1c" in test_name or \ + (u"-1c-" in test_name and + testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")): + cores = u"1t1c" + elif u"2t2c" in test_name or \ + (u"-2c-" in test_name and + testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")): + cores = u"2t2c" + elif u"4t4c" in test_name or \ + (u"-4c-" in test_name and + testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")): + cores = u"4t4c" + elif u"2t1c" in test_name or \ + (u"-1c-" in test_name and + testbed in (u"2n-skx", u"3n-skx", u"2n-clx")): + cores = u"2t1c" + elif u"4t2c" in test_name or \ + (u"-2c-" in test_name and + testbed in (u"2n-skx", u"3n-skx", u"2n-clx")): + cores = u"4t2c" + elif u"8t4c" in test_name or \ + (u"-4c-" in test_name and + testbed in (u"2n-skx", u"3n-skx", u"2n-clx")): + cores = u"8t4c" + else: + cores = u"" + + if u"testpmd" in test_name: + driver = u"testpmd" + elif u"l3fwd" in test_name: + driver = u"l3fwd" + elif u"avf" in test_name: + driver = u"avf" + elif u"rdma" in test_name: + driver = u"rdma" + elif u"dnv" in testbed or u"tsh" in testbed: + driver = u"ixgbe" + else: + driver = u"dpdk" + + if u"acl" in test_name or \ + u"macip" in test_name or \ + u"nat" in test_name or \ + u"policer" in test_name or \ + u"cop" in test_name: + bsf = u"features" + elif u"scale" in test_name: + bsf = u"scale" + elif u"base" in test_name: + bsf = u"base" + else: + bsf = u"base" + + if u"114b" in test_name and u"vhost" in test_name: + domain = u"vts" + elif u"testpmd" in test_name or u"l3fwd" in test_name: + domain = u"dpdk" + elif u"memif" in test_name: + domain = u"container_memif" + elif u"srv6" in test_name: + domain = u"srv6" + elif u"vhost" in test_name: + domain = u"vhost" + if u"vppl2xc" in test_name: + driver += u"-vpp" + else: + driver += u"-testpmd" + if u"lbvpplacp" in test_name: + bsf += u"-link-bonding" + elif u"ch" in test_name and u"vh" in test_name and u"vm" in test_name: + domain = u"nf_service_density_vnfc" + elif u"ch" in test_name and u"mif" in test_name and u"dcr" in test_name: + domain = u"nf_service_density_cnfc" + elif u"pl" in test_name and u"mif" in test_name and u"dcr" in test_name: + domain = u"nf_service_density_cnfp" + elif u"ipsec" in test_name: + domain = u"ipsec" + if u"sw" in test_name: + bsf += u"-sw" + elif u"hw" in test_name: + bsf += u"-hw" + elif u"ethip4vxlan" in test_name: + domain = u"ip4_tunnels" + elif u"ip4base" in test_name or u"ip4scale" in test_name: + domain = u"ip4" + elif u"ip6base" in test_name or u"ip6scale" in test_name: + domain = u"ip6" + elif u"l2xcbase" in test_name or \ + u"l2xcscale" in test_name or \ + u"l2bdbasemaclrn" in test_name or \ + u"l2bdscale" in test_name or \ + u"l2patch" in test_name: + domain = u"l2" + else: + domain = u"" - elif "testpmd" in test_name or "l3fwd" in test_name: - file_name = "dpdk" + file_name = u"-".join((domain, testbed, nic)) + u".html#" + anchor_name = u"-".join((frame_size, cores, bsf, driver)) - elif "memif" in test_name: - file_name = "container_memif" - feature = "-base" + return file_name + anchor_name - elif "srv6" in test_name: - file_name = "srv6" - elif "vhost" in test_name: - if "l2xcbase" in test_name or "l2bdbasemaclrn" in test_name: - file_name = "vm_vhost_l2" - if "114b" in test_name: - feature = "" - elif "l2xcbase" in test_name and "x520" in test_name: - feature = "-base-l2xc" - elif "l2bdbasemaclrn" in test_name and "x520" in test_name: - feature = "-base-l2bd" - else: - feature = "-base" - elif "ip4base" in test_name: - file_name = "vm_vhost_ip4" - feature = "-base" - - elif "ipsecbasetnlsw" in test_name: - file_name = "ipsecsw" - feature = "-base-scale" - - elif "ipsec" in test_name: - file_name = "ipsec" - feature = "-base-scale" - if "hw-" in test_name: - file_name = "ipsechw" - elif "sw-" in test_name: - file_name = "ipsecsw" - if "-int-" in test_name: - feature = "-base-scale-int" - elif "tnl" in test_name: - feature = "-base-scale-tnl" - - elif "ethip4lispip" in test_name or "ethip4vxlan" in test_name: - file_name = "ip4_tunnels" - feature = "-base" - - elif "ip4base" in test_name or "ip4scale" in test_name: - file_name = "ip4" - if "xl710" in test_name: - feature = "-base-scale-features" - elif "iacl" in test_name: - feature = "-features-iacl" - elif "oacl" in test_name: - feature = "-features-oacl" - elif "snat" in test_name or "cop" in test_name: - feature = "-features" - else: - feature = "-base-scale" - - elif "ip6base" in test_name or "ip6scale" in test_name: - file_name = "ip6" - feature = "-base-scale" - - elif "l2xcbase" in test_name or "l2xcscale" in test_name \ - or "l2bdbasemaclrn" in test_name or "l2bdscale" in test_name \ - or "l2dbbasemaclrn" in test_name or "l2dbscale" in test_name: - file_name = "l2" - if "macip" in test_name: - feature = "-features-macip" - elif "iacl" in test_name: - feature = "-features-iacl" - elif "oacl" in test_name: - feature = "-features-oacl" - else: - feature = "-base-scale" - - if "x520" in test_name: - nic = "x520-" - elif "x710" in test_name: - nic = "x710-" - elif "xl710" in test_name: - nic = "xl710-" - elif "xxv710" in test_name: - nic = "xxv710-" - elif "vic1227" in test_name: - nic = "vic1227-" - elif "vic1385" in test_name: - nic = "vic1385-" - elif "x553" in test_name: - nic = "x553-" - else: - nic = "" - anchor += nic - - if "64b" in test_name: - framesize = "64b" - elif "78b" in test_name: - framesize = "78b" - elif "imix" in test_name: - framesize = "imix" - elif "9000b" in test_name: - framesize = "9000b" - elif "1518b" in test_name: - framesize = "1518b" - elif "114b" in test_name: - framesize = "114b" - else: - framesize = "" - anchor += framesize + '-' - - if "1t1c" in test_name: - anchor += "1t1c" - elif "2t2c" in test_name: - anchor += "2t2c" - elif "4t4c" in test_name: - anchor += "4t4c" - elif "2t1c" in test_name: - anchor += "2t1c" - elif "4t2c" in test_name: - anchor += "4t2c" - elif "8t4c" in test_name: - anchor += "8t4c" - - return url + file_name + '-' + testbed + '-' + nic + framesize + \ - feature.replace("-int", "").replace("-tnl", "") + anchor + feature - - -def table_performance_trending_dashboard_html(table, input_data): +def table_perf_trending_dash_html(table, input_data): """Generate the table(s) with algorithm: - table_performance_trending_dashboard_html specified in the specification + table_perf_trending_dash_html specified in the specification file. :param table: Table to generate. @@ -1236,70 +1769,94 @@ def table_performance_trending_dashboard_html(table, input_data): :type input_data: InputData """ - testbed = table.get("testbed", None) - if testbed is None: - logging.error("The testbed is not defined for the table '{0}'.". - format(table.get("title", ""))) + _ = input_data + + if not table.get(u"testbed", None): + logging.error( + f"The testbed is not defined for the table " + f"{table.get(u'title', u'')}." + ) return - logging.info(" Generating the table {0} ...". - format(table.get("title", ""))) + logging.info(f" Generating the table {table.get(u'title', u'')} ...") try: - with open(table["input-file"], 'rb') as csv_file: - csv_content = csv.reader(csv_file, delimiter=',', quotechar='"') - csv_lst = [item for item in csv_content] + with open(table[u"input-file"], u'rt') as csv_file: + csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"')) except KeyError: - logging.warning("The input file is not defined.") + logging.warning(u"The input file is not defined.") return except csv.Error as err: - logging.warning("Not possible to process the file '{0}'.\n{1}". - format(table["input-file"], err)) + logging.warning( + f"Not possible to process the file {table[u'input-file']}.\n" + f"{repr(err)}" + ) return # Table: - dashboard = ET.Element("table", attrib=dict(width="100%", border='0')) + dashboard = ET.Element(u"table", attrib=dict(width=u"100%", border=u'0')) # Table header: - tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor="#7eade7")) + trow = ET.SubElement(dashboard, u"tr", attrib=dict(bgcolor=u"#7eade7")) for idx, item in enumerate(csv_lst[0]): - alignment = "left" if idx == 0 else "center" - th = ET.SubElement(tr, "th", attrib=dict(align=alignment)) - th.text = item + alignment = u"left" if idx == 0 else u"center" + thead = ET.SubElement(trow, u"th", attrib=dict(align=alignment)) + thead.text = item # Rows: - colors = {"regression": ("#ffcccc", "#ff9999"), - "progression": ("#c6ecc6", "#9fdf9f"), - "normal": ("#e9f1fb", "#d4e4f7")} + colors = { + u"regression": ( + u"#ffcccc", + u"#ff9999" + ), + u"progression": ( + u"#c6ecc6", + u"#9fdf9f" + ), + u"normal": ( + u"#e9f1fb", + u"#d4e4f7" + ) + } for r_idx, row in enumerate(csv_lst[1:]): if int(row[4]): - color = "regression" + color = u"regression" elif int(row[5]): - color = "progression" + color = u"progression" else: - color = "normal" - background = colors[color][r_idx % 2] - tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor=background)) + color = u"normal" + trow = ET.SubElement( + dashboard, u"tr", attrib=dict(bgcolor=colors[color][r_idx % 2]) + ) # Columns: for c_idx, item in enumerate(row): - alignment = "left" if c_idx == 0 else "center" - td = ET.SubElement(tr, "td", attrib=dict(align=alignment)) + tdata = ET.SubElement( + trow, + u"td", + attrib=dict(align=u"left" if c_idx == 0 else u"center") + ) # Name: if c_idx == 0: - url = _generate_url("../trending/", testbed, item) - ref = ET.SubElement(td, "a", attrib=dict(href=url)) + ref = ET.SubElement( + tdata, + u"a", + attrib=dict( + href=f"../trending/" + f"{_generate_url(table.get(u'testbed', ''), item)}" + ) + ) ref.text = item else: - td.text = item + tdata.text = item try: - with open(table["output-file"], 'w') as html_file: - logging.info(" Writing file: '{0}'".format(table["output-file"])) - html_file.write(".. raw:: html\n\n\t") - html_file.write(ET.tostring(dashboard)) - html_file.write("\n\t



\n") + with open(table[u"output-file"], u'w') as html_file: + logging.info(f" Writing file: {table[u'output-file']}") + html_file.write(u".. raw:: html\n\n\t") + html_file.write(str(ET.tostring(dashboard, encoding=u"unicode"))) + html_file.write(u"\n\t



\n") except KeyError: - logging.warning("The output file is not defined.") + logging.warning(u"The output file is not defined.") return @@ -1313,45 +1870,56 @@ def table_last_failed_tests(table, input_data): :type input_data: InputData """ - logging.info(" Generating the table {0} ...". - format(table.get("title", ""))) + logging.info(f" Generating the table {table.get(u'title', u'')} ...") # Transform the data - logging.info(" Creating the data set for the {0} '{1}'.". - format(table.get("type", ""), table.get("title", ""))) + logging.info( + f" Creating the data set for the {table.get(u'type', u'')} " + f"{table.get(u'title', u'')}." + ) + data = input_data.filter_data(table, continue_on_error=True) if data is None or data.empty: - logging.warn(" No data for the {0} '{1}'.". - format(table.get("type", ""), table.get("title", ""))) + logging.warning( + f" No data for the {table.get(u'type', u'')} " + f"{table.get(u'title', u'')}." + ) return tbl_list = list() - for job, builds in table["data"].items(): + for job, builds in table[u"data"].items(): for build in builds: build = str(build) try: - version = input_data.metadata(job, build).get("version", "") + version = input_data.metadata(job, build).get(u"version", u"") except KeyError: - logging.error("Data for {job}: {build} is not present.". - format(job=job, build=build)) + logging.error(f"Data for {job}: {build} is not present.") return tbl_list.append(build) tbl_list.append(version) - for tst_name, tst_data in data[job][build].iteritems(): - if tst_data["status"] != "FAIL": + failed_tests = list() + passed = 0 + failed = 0 + for tst_data in data[job][build].values: + if tst_data[u"status"] != u"FAIL": + passed += 1 continue - groups = re.search(REGEX_NIC, tst_data["parent"]) + failed += 1 + groups = re.search(REGEX_NIC, tst_data[u"parent"]) if not groups: continue nic = groups.group(0) - tbl_list.append("{0}-{1}".format(nic, tst_data["name"])) - - file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"]) - logging.info(" Writing file: '{0}'".format(file_name)) - with open(file_name, "w") as file_handler: + failed_tests.append(f"{nic}-{tst_data[u'name']}") + tbl_list.append(str(passed)) + tbl_list.append(str(failed)) + tbl_list.extend(failed_tests) + + file_name = f"{table[u'output-file']}{table[u'output-file-ext']}" + logging.info(f" Writing file: {file_name}") + with open(file_name, u"wt") as file_handler: for test in tbl_list: - file_handler.write(test + '\n') + file_handler.write(test + u'\n') def table_failed_tests(table, input_data): @@ -1364,92 +1932,103 @@ def table_failed_tests(table, input_data): :type input_data: InputData """ - logging.info(" Generating the table {0} ...". - format(table.get("title", ""))) + logging.info(f" Generating the table {table.get(u'title', u'')} ...") # Transform the data - logging.info(" Creating the data set for the {0} '{1}'.". - format(table.get("type", ""), table.get("title", ""))) + logging.info( + f" Creating the data set for the {table.get(u'type', u'')} " + f"{table.get(u'title', u'')}." + ) data = input_data.filter_data(table, continue_on_error=True) # Prepare the header of the tables - header = ["Test Case", - "Failures [#]", - "Last Failure [Time]", - "Last Failure [VPP-Build-Id]", - "Last Failure [CSIT-Job-Build-Id]"] + header = [ + u"Test Case", + u"Failures [#]", + u"Last Failure [Time]", + u"Last Failure [VPP-Build-Id]", + u"Last Failure [CSIT-Job-Build-Id]" + ] # Generate the data for the table according to the model in the table # specification now = dt.utcnow() - timeperiod = timedelta(int(table.get("window", 7))) + timeperiod = timedelta(int(table.get(u"window", 7))) tbl_dict = dict() - for job, builds in table["data"].items(): + for job, builds in table[u"data"].items(): for build in builds: build = str(build) - for tst_name, tst_data in data[job][build].iteritems(): - if tst_name.lower() in table.get("ignore-list", list()): + for tst_name, tst_data in data[job][build].items(): + if tst_name.lower() in table.get(u"ignore-list", list()): continue if tbl_dict.get(tst_name, None) is None: - groups = re.search(REGEX_NIC, tst_data["parent"]) + groups = re.search(REGEX_NIC, tst_data[u"parent"]) if not groups: continue nic = groups.group(0) tbl_dict[tst_name] = { - "name": "{0}-{1}".format(nic, tst_data["name"]), - "data": OrderedDict()} + u"name": f"{nic}-{tst_data[u'name']}", + u"data": OrderedDict() + } try: generated = input_data.metadata(job, build).\ - get("generated", "") + get(u"generated", u"") if not generated: continue - then = dt.strptime(generated, "%Y%m%d %H:%M") + then = dt.strptime(generated, u"%Y%m%d %H:%M") if (now - then) <= timeperiod: - tbl_dict[tst_name]["data"][build] = ( - tst_data["status"], + tbl_dict[tst_name][u"data"][build] = ( + tst_data[u"status"], generated, - input_data.metadata(job, build).get("version", ""), - build) + input_data.metadata(job, build).get(u"version", + u""), + build + ) except (TypeError, KeyError) as err: - logging.warning("tst_name: {} - err: {}". - format(tst_name, repr(err))) + logging.warning(f"tst_name: {tst_name} - err: {repr(err)}") max_fails = 0 tbl_lst = list() for tst_data in tbl_dict.values(): fails_nr = 0 - for val in tst_data["data"].values(): - if val[0] == "FAIL": + fails_last_date = u"" + fails_last_vpp = u"" + fails_last_csit = u"" + for val in tst_data[u"data"].values(): + if val[0] == u"FAIL": fails_nr += 1 fails_last_date = val[1] fails_last_vpp = val[2] fails_last_csit = val[3] if fails_nr: max_fails = fails_nr if fails_nr > max_fails else max_fails - tbl_lst.append([tst_data["name"], - fails_nr, - fails_last_date, - fails_last_vpp, - "mrr-daily-build-{0}".format(fails_last_csit)]) + tbl_lst.append( + [ + tst_data[u"name"], + fails_nr, + fails_last_date, + fails_last_vpp, + f"mrr-daily-build-{fails_last_csit}" + ] + ) tbl_lst.sort(key=lambda rel: rel[2], reverse=True) tbl_sorted = list() for nrf in range(max_fails, -1, -1): tbl_fails = [item for item in tbl_lst if item[1] == nrf] tbl_sorted.extend(tbl_fails) - file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"]) - logging.info(" Writing file: '{0}'".format(file_name)) - with open(file_name, "w") as file_handler: - file_handler.write(",".join(header) + "\n") + file_name = f"{table[u'output-file']}{table[u'output-file-ext']}" + logging.info(f" Writing file: {file_name}") + with open(file_name, u"wt") as file_handler: + file_handler.write(u",".join(header) + u"\n") for test in tbl_sorted: - file_handler.write(",".join([str(item) for item in test]) + '\n') + file_handler.write(u",".join([str(item) for item in test]) + u'\n') - txt_file_name = "{0}.txt".format(table["output-file"]) - logging.info(" Writing file: '{0}'".format(txt_file_name)) - convert_csv_to_pretty_txt(file_name, txt_file_name) + logging.info(f" Writing file: {table[u'output-file']}.txt") + convert_csv_to_pretty_txt(file_name, f"{table[u'output-file']}.txt") def table_failed_tests_html(table, input_data): @@ -1462,60 +2041,74 @@ def table_failed_tests_html(table, input_data): :type input_data: InputData """ - testbed = table.get("testbed", None) - if testbed is None: - logging.error("The testbed is not defined for the table '{0}'.". - format(table.get("title", ""))) + _ = input_data + + if not table.get(u"testbed", None): + logging.error( + f"The testbed is not defined for the table " + f"{table.get(u'title', u'')}." + ) return - logging.info(" Generating the table {0} ...". - format(table.get("title", ""))) + logging.info(f" Generating the table {table.get(u'title', u'')} ...") try: - with open(table["input-file"], 'rb') as csv_file: - csv_content = csv.reader(csv_file, delimiter=',', quotechar='"') - csv_lst = [item for item in csv_content] + with open(table[u"input-file"], u'rt') as csv_file: + csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"')) except KeyError: - logging.warning("The input file is not defined.") + logging.warning(u"The input file is not defined.") return except csv.Error as err: - logging.warning("Not possible to process the file '{0}'.\n{1}". - format(table["input-file"], err)) + logging.warning( + f"Not possible to process the file {table[u'input-file']}.\n" + f"{repr(err)}" + ) return # Table: - failed_tests = ET.Element("table", attrib=dict(width="100%", border='0')) + failed_tests = ET.Element(u"table", attrib=dict(width=u"100%", border=u'0')) # Table header: - tr = ET.SubElement(failed_tests, "tr", attrib=dict(bgcolor="#7eade7")) + trow = ET.SubElement(failed_tests, u"tr", attrib=dict(bgcolor=u"#7eade7")) for idx, item in enumerate(csv_lst[0]): - alignment = "left" if idx == 0 else "center" - th = ET.SubElement(tr, "th", attrib=dict(align=alignment)) - th.text = item + alignment = u"left" if idx == 0 else u"center" + thead = ET.SubElement(trow, u"th", attrib=dict(align=alignment)) + thead.text = item # Rows: - colors = ("#e9f1fb", "#d4e4f7") + colors = (u"#e9f1fb", u"#d4e4f7") for r_idx, row in enumerate(csv_lst[1:]): background = colors[r_idx % 2] - tr = ET.SubElement(failed_tests, "tr", attrib=dict(bgcolor=background)) + trow = ET.SubElement( + failed_tests, u"tr", attrib=dict(bgcolor=background) + ) # Columns: for c_idx, item in enumerate(row): - alignment = "left" if c_idx == 0 else "center" - td = ET.SubElement(tr, "td", attrib=dict(align=alignment)) + tdata = ET.SubElement( + trow, + u"td", + attrib=dict(align=u"left" if c_idx == 0 else u"center") + ) # Name: if c_idx == 0: - url = _generate_url("../trending/", testbed, item) - ref = ET.SubElement(td, "a", attrib=dict(href=url)) + ref = ET.SubElement( + tdata, + u"a", + attrib=dict( + href=f"../trending/" + f"{_generate_url(table.get(u'testbed', ''), item)}" + ) + ) ref.text = item else: - td.text = item + tdata.text = item try: - with open(table["output-file"], 'w') as html_file: - logging.info(" Writing file: '{0}'".format(table["output-file"])) - html_file.write(".. raw:: html\n\n\t") - html_file.write(ET.tostring(failed_tests)) - html_file.write("\n\t



\n") + with open(table[u"output-file"], u'w') as html_file: + logging.info(f" Writing file: {table[u'output-file']}") + html_file.write(u".. raw:: html\n\n\t") + html_file.write(str(ET.tostring(failed_tests, encoding=u"unicode"))) + html_file.write(u"\n\t



\n") except KeyError: - logging.warning("The output file is not defined.") + logging.warning(u"The output file is not defined.") return