X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=blobdiff_plain;f=resources%2Ftools%2Fpresentation%2Fgenerator_tables.py;h=4cbc7c074647b0706d95694de2689ac4c08956e0;hp=3c77ea89f1ae3978ef59fecb9251515668fd3626;hb=b95d8570e5edb080d7cfc001daed66b47811068a;hpb=0ba8b76dc5ac35b39b1a9200c1c81410d632d8aa diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py index 3c77ea89f1..4cbc7c0746 100644 --- a/resources/tools/presentation/generator_tables.py +++ b/resources/tools/presentation/generator_tables.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019 Cisco and/or its affiliates. +# Copyright (c) 2020 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -29,12 +29,13 @@ import plotly.offline as ploff import pandas as pd from numpy import nan, isnan +from yaml import load, FullLoader, YAMLError -from pal_utils import mean, stdev, relative_change, classify_anomalies, \ +from pal_utils import mean, stdev, classify_anomalies, \ convert_csv_to_pretty_txt, relative_change_stdev -REGEX_NIC = re.compile(r'(\d*ge\dp\d\D*\d*[a-z]*)-') +REGEX_NIC = re.compile(r'(\d*ge\dp\d\D*\d*[a-z]*)') def generate_tables(spec, data): @@ -47,7 +48,6 @@ def generate_tables(spec, data): """ generator = { - u"table_details": table_details, u"table_merged_details": table_merged_details, u"table_perf_comparison": table_perf_comparison, u"table_perf_comparison_nic": table_perf_comparison_nic, @@ -97,7 +97,14 @@ def table_oper_data_html(table, input_data): if data.empty: return data = input_data.merge_data(data) - data.sort_index(inplace=True) + + sort_tests = table.get(u"sort", None) + if sort_tests: + args = dict( + inplace=True, + ascending=(sort_tests == u"ascending") + ) + data.sort_index(**args) suites = input_data.filter_data( table, @@ -145,6 +152,17 @@ def table_oper_data_html(table, input_data): trow, u"td", attrib=dict(align=u"left", colspan=u"6") ) tcol.text = u"No Data" + + trow = ET.SubElement( + tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]) + ) + thead = ET.SubElement( + trow, u"th", attrib=dict(align=u"left", colspan=u"6") + ) + font = ET.SubElement( + thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff") + ) + font.text = u"." return str(ET.tostring(tbl, encoding=u"unicode")) tbl_hdr = ( @@ -156,7 +174,7 @@ def table_oper_data_html(table, input_data): u"Average Vector Size" ) - for dut_name, dut_data in tst_data[u"show-run"].items(): + for dut_data in tst_data[u"show-run"].values(): trow = ET.SubElement( tbl, u"tr", attrib=dict(bgcolor=colors[u"header"]) ) @@ -166,15 +184,7 @@ def table_oper_data_html(table, input_data): if dut_data.get(u"threads", None) is None: tcol.text = u"No Data" continue - bold = ET.SubElement(tcol, u"b") - bold.text = dut_name - trow = ET.SubElement( - tbl, u"tr", attrib=dict(bgcolor=colors[u"body"][0]) - ) - tcol = ET.SubElement( - trow, u"td", attrib=dict(align=u"left", colspan=u"6") - ) bold = ET.SubElement(tcol, u"b") bold.text = ( f"Host IP: {dut_data.get(u'host', '')}, " @@ -255,7 +265,7 @@ def table_oper_data_html(table, input_data): if not html_table: continue try: - file_name = f"{table[u'output-file']}_{suite[u'name']}.rst" + file_name = f"{table[u'output-file']}{suite[u'name']}.rst" with open(f"{file_name}", u'w') as html_file: logging.info(f" Writing file: {file_name}") html_file.write(u".. raw:: html\n\n\t") @@ -267,80 +277,6 @@ def table_oper_data_html(table, input_data): logging.info(u" Done.") -def table_details(table, input_data): - """Generate the table(s) with algorithm: table_detailed_test_results - specified in the specification file. - - :param table: Table to generate. - :param input_data: Data to process. - :type table: pandas.Series - :type input_data: InputData - """ - - logging.info(f" Generating the table {table.get(u'title', u'')} ...") - - # Transform the data - logging.info( - f" Creating the data set for the {table.get(u'type', u'')} " - f"{table.get(u'title', u'')}." - ) - data = input_data.filter_data(table) - - # Prepare the header of the tables - header = list() - for column in table[u"columns"]: - header.append( - u'"{0}"'.format(str(column[u"title"]).replace(u'"', u'""')) - ) - - # Generate the data for the table according to the model in the table - # specification - job = list(table[u"data"].keys())[0] - build = str(table[u"data"][job][0]) - try: - suites = input_data.suites(job, build) - except KeyError: - logging.error( - u" No data available. The table will not be generated." - ) - return - - for suite in suites.values: - # Generate data - suite_name = suite[u"name"] - table_lst = list() - for test in data[job][build].keys(): - if data[job][build][test][u"parent"] not in suite_name: - continue - row_lst = list() - for column in table[u"columns"]: - try: - col_data = str(data[job][build][test][column[ - u"data"].split(" ")[1]]).replace(u'"', u'""') - if column[u"data"].split(u" ")[1] in \ - (u"conf-history", u"show-run"): - col_data = col_data.replace(u" |br| ", u"", 1) - col_data = f" |prein| {col_data[:-5]} |preout| " - row_lst.append(f'"{col_data}"') - except KeyError: - row_lst.append(u"No data") - table_lst.append(row_lst) - - # Write the data to file - if table_lst: - file_name = ( - f"{table[u'output-file']}_{suite_name}" - f"{table[u'output-file-ext']}" - ) - logging.info(f" Writing file: {file_name}") - with open(file_name, u"wt") as file_handler: - file_handler.write(u",".join(header) + u"\n") - for item in table_lst: - file_handler.write(u",".join(item) + u"\n") - - logging.info(u" Done.") - - def table_merged_details(table, input_data): """Generate the table(s) with algorithm: table_merged_details specified in the specification file. @@ -352,6 +288,7 @@ def table_merged_details(table, input_data): """ logging.info(f" Generating the table {table.get(u'title', u'')} ...") + # Transform the data logging.info( f" Creating the data set for the {table.get(u'type', u'')} " @@ -359,12 +296,15 @@ def table_merged_details(table, input_data): ) data = input_data.filter_data(table, continue_on_error=True) data = input_data.merge_data(data) - data.sort_index(inplace=True) - logging.info( - f" Creating the data set for the {table.get(u'type', u'')} " - f"{table.get(u'title', u'')}." - ) + sort_tests = table.get(u"sort", None) + if sort_tests: + args = dict( + inplace=True, + ascending=(sort_tests == u"ascending") + ) + data.sort_index(**args) + suites = input_data.filter_data( table, continue_on_error=True, data_set=u"suites") suites = input_data.merge_data(suites) @@ -388,24 +328,42 @@ def table_merged_details(table, input_data): try: col_data = str(data[test][column[ u"data"].split(u" ")[1]]).replace(u'"', u'""') + # Do not include tests with "Test Failed" in test message + if u"Test Failed" in col_data: + continue col_data = col_data.replace( u"No Data", u"Not Captured " ) - if column[u"data"].split(u" ")[1] in \ - (u"conf-history", u"show-run"): + if column[u"data"].split(u" ")[1] in (u"name", ): + if len(col_data) > 30: + col_data_lst = col_data.split(u"-") + half = int(len(col_data_lst) / 2) + col_data = f"{u'-'.join(col_data_lst[:half])}" \ + f"- |br| " \ + f"{u'-'.join(col_data_lst[half:])}" + col_data = f" |prein| {col_data} |preout| " + elif column[u"data"].split(u" ")[1] in (u"msg", ): + # Temporary solution: remove NDR results from message: + if bool(table.get(u'remove-ndr', False)): + try: + col_data = col_data.split(u" |br| ", 1)[1] + except IndexError: + pass + col_data = f" |prein| {col_data} |preout| " + elif column[u"data"].split(u" ")[1] in \ + (u"conf-history", u"show-run"): col_data = col_data.replace(u" |br| ", u"", 1) col_data = f" |prein| {col_data[:-5]} |preout| " row_lst.append(f'"{col_data}"') except KeyError: row_lst.append(u'"Not captured"') - table_lst.append(row_lst) + if len(row_lst) == len(table[u"columns"]): + table_lst.append(row_lst) # Write the data to file if table_lst: - file_name = ( - f"{table[u'output-file']}_{suite_name}" - f"{table[u'output-file-ext']}" - ) + separator = u"" if table[u'output-file'].endswith(u"/") else u"_" + file_name = f"{table[u'output-file']}{separator}{suite_name}.csv" logging.info(f" Writing file: {file_name}") with open(file_name, u"wt") as file_handler: file_handler.write(u",".join(header) + u"\n") @@ -469,7 +427,12 @@ def _tpc_insert_data(target, src, include_tests): """ try: if include_tests == u"MRR": - target.append(src[u"result"][u"receive-rate"]) + target.append( + ( + src[u"result"][u"receive-rate"], + src[u"result"][u"receive-stdev"] + ) + ) elif include_tests == u"PDR": target.append(src[u"throughput"][u"PDR"][u"LOWER"]) elif include_tests == u"NDR": @@ -491,7 +454,6 @@ def _tpc_sort_table(table): :rtype: list """ - tbl_new = list() tbl_see = list() tbl_delta = list() @@ -507,12 +469,14 @@ def _tpc_sort_table(table): # Sort the tables: tbl_new.sort(key=lambda rel: rel[0], reverse=False) tbl_see.sort(key=lambda rel: rel[0], reverse=False) - tbl_see.sort(key=lambda rel: rel[-1], reverse=False) - tbl_delta.sort(key=lambda rel: rel[-1], reverse=True) + tbl_see.sort(key=lambda rel: rel[-2], reverse=False) + tbl_delta.sort(key=lambda rel: rel[0], reverse=False) + tbl_delta.sort(key=lambda rel: rel[-2], reverse=True) # Put the tables together: table = list() - table.extend(tbl_new) + # We do not want "New in CSIT": + # table.extend(tbl_new) table.extend(tbl_see) table.extend(tbl_delta) @@ -534,14 +498,24 @@ def _tpc_generate_html_table(header, data, output_file_name): :type output_file_name: str """ + try: + idx = header.index(u"Test case") + except ValueError: + idx = 0 + params = { + u"align-hdr": ([u"left", u"center"], [u"left", u"left", u"center"]), + u"align-itm": ([u"left", u"right"], [u"left", u"left", u"right"]), + u"width": ([28, 9], [4, 24, 10]) + } + df_data = pd.DataFrame(data, columns=header) df_sorted = [df_data.sort_values( - by=[key, header[0]], ascending=[True, True] - if key != header[0] else [False, True]) for key in header] + by=[key, header[idx]], ascending=[True, True] + if key != header[idx] else [False, True]) for key in header] df_sorted_rev = [df_data.sort_values( - by=[key, header[0]], ascending=[False, True] - if key != header[0] else [True, True]) for key in header] + by=[key, header[idx]], ascending=[False, True] + if key != header[idx] else [True, True]) for key in header] df_sorted.extend(df_sorted_rev) fill_color = [[u"#d4e4f7" if idx % 2 else u"#e9f1fb" @@ -549,7 +523,7 @@ def _tpc_generate_html_table(header, data, output_file_name): table_header = dict( values=[f"{item}" for item in header], fill_color=u"#7eade7", - align=[u"left", u"center"] + align=params[u"align-hdr"][idx] ) fig = go.Figure() @@ -558,12 +532,12 @@ def _tpc_generate_html_table(header, data, output_file_name): columns = [table.get(col) for col in header] fig.add_trace( go.Table( - columnwidth=[30, 10], + columnwidth=params[u"width"][idx], header=table_header, cells=dict( values=columns, fill_color=fill_color, - align=[u"left", u"right"] + align=params[u"align-itm"][idx] ) ) ) @@ -635,6 +609,16 @@ def table_perf_comparison(table, input_data): try: header = [u"Test case", ] + rca_data = None + rca = table.get(u"rca", None) + if rca: + try: + with open(rca.get(u"data-file", ""), u"r") as rca_file: + rca_data = load(rca_file, Loader=FullLoader) + header.insert(0, rca.get(u"title", "RCA")) + except (YAMLError, IOError) as err: + logging.warning(repr(err)) + if table[u"include-tests"] == u"MRR": hdr_param = u"Rec Rate" else: @@ -654,23 +638,24 @@ def table_perf_comparison(table, input_data): f"{table[u'reference'][u'title']} Stdev [Mpps]", f"{table[u'compare'][u'title']} {hdr_param} [Mpps]", f"{table[u'compare'][u'title']} Stdev [Mpps]", - u"Delta [%]" + u"Delta [%]", + u"Stdev of delta [%]" ] ) - header_str = u",".join(header) + u"\n" + header_str = u";".join(header) + u"\n" except (AttributeError, KeyError) as err: logging.error(f"The model is invalid, missing parameter: {repr(err)}") return # Prepare data to the table: tbl_dict = dict() - # topo = "" for job, builds in table[u"reference"][u"data"].items(): - # topo = u"2n-skx" if u"2n-skx" in job else u"" for build in builds: for tst_name, tst_data in data[job][str(build)].items(): tst_name_mod = _tpc_modify_test_name(tst_name) - if u"across topologies" in table[u"title"].lower(): + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: groups = re.search(REGEX_NIC, tst_data[u"parent"]) @@ -698,7 +683,9 @@ def table_perf_comparison(table, input_data): for build in builds: for tst_name, tst_data in rpl_data[job][str(build)].items(): tst_name_mod = _tpc_modify_test_name(tst_name) - if u"across topologies" in table[u"title"].lower(): + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: name = \ @@ -725,7 +712,9 @@ def table_perf_comparison(table, input_data): for build in builds: for tst_name, tst_data in data[job][str(build)].items(): tst_name_mod = _tpc_modify_test_name(tst_name) - if u"across topologies" in table[u"title"].lower(): + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: groups = re.search(REGEX_NIC, tst_data[u"parent"]) @@ -755,7 +744,9 @@ def table_perf_comparison(table, input_data): for build in builds: for tst_name, tst_data in rpl_data[job][str(build)].items(): tst_name_mod = _tpc_modify_test_name(tst_name) - if u"across topologies" in table[u"title"].lower(): + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: name = \ @@ -783,7 +774,9 @@ def table_perf_comparison(table, input_data): for build in builds: for tst_name, tst_data in data[job][str(build)].items(): tst_name_mod = _tpc_modify_test_name(tst_name) - if u"across topologies" in table[u"title"].lower(): + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: continue @@ -795,7 +788,8 @@ def table_perf_comparison(table, input_data): u"title"]] = list() try: if table[u"include-tests"] == u"MRR": - res = tst_data[u"result"][u"receive-rate"] + res = (tst_data[u"result"][u"receive-rate"], + tst_data[u"result"][u"receive-stdev"]) elif table[u"include-tests"] == u"PDR": res = tst_data[u"throughput"][u"PDR"][u"LOWER"] elif table[u"include-tests"] == u"NDR": @@ -808,41 +802,71 @@ def table_perf_comparison(table, input_data): pass tbl_lst = list() - footnote = False for tst_name in tbl_dict: item = [tbl_dict[tst_name][u"name"], ] if history: if tbl_dict[tst_name].get(u"history", None) is not None: for hist_data in tbl_dict[tst_name][u"history"].values(): if hist_data: - item.append(round(mean(hist_data) / 1000000, 2)) - item.append(round(stdev(hist_data) / 1000000, 2)) + if table[u"include-tests"] == u"MRR": + item.append(round(hist_data[0][0] / 1e6, 2)) + item.append(round(hist_data[0][1] / 1e6, 2)) + else: + item.append(round(mean(hist_data) / 1e6, 2)) + item.append(round(stdev(hist_data) / 1e6, 2)) else: item.extend([u"Not tested", u"Not tested"]) else: item.extend([u"Not tested", u"Not tested"]) - data_t = tbl_dict[tst_name][u"ref-data"] - if data_t: - item.append(round(mean(data_t) / 1000000, 2)) - item.append(round(stdev(data_t) / 1000000, 2)) + data_r = tbl_dict[tst_name][u"ref-data"] + if data_r: + if table[u"include-tests"] == u"MRR": + data_r_mean = data_r[0][0] + data_r_stdev = data_r[0][1] + else: + data_r_mean = mean(data_r) + data_r_stdev = stdev(data_r) + item.append(round(data_r_mean / 1e6, 2)) + item.append(round(data_r_stdev / 1e6, 2)) else: + data_r_mean = None + data_r_stdev = None item.extend([u"Not tested", u"Not tested"]) - data_t = tbl_dict[tst_name][u"cmp-data"] - if data_t: - item.append(round(mean(data_t) / 1000000, 2)) - item.append(round(stdev(data_t) / 1000000, 2)) + data_c = tbl_dict[tst_name][u"cmp-data"] + if data_c: + if table[u"include-tests"] == u"MRR": + data_c_mean = data_c[0][0] + data_c_stdev = data_c[0][1] + else: + data_c_mean = mean(data_c) + data_c_stdev = stdev(data_c) + item.append(round(data_c_mean / 1e6, 2)) + item.append(round(data_c_stdev / 1e6, 2)) else: + data_c_mean = None + data_c_stdev = None item.extend([u"Not tested", u"Not tested"]) if item[-2] == u"Not tested": pass elif item[-4] == u"Not tested": item.append(u"New in CSIT-2001") - # elif topo == u"2n-skx" and u"dot1q" in tbl_dict[tst_name][u"name"]: - # item.append(u"See footnote [1]") - # footnote = True - elif item[-4] != 0: - item.append(int(relative_change(float(item[-4]), float(item[-2])))) - if (len(item) == len(header)) and (item[-3] != u"Not tested"): + item.append(u"New in CSIT-2001") + elif data_r_mean is not None and data_c_mean is not None: + delta, d_stdev = relative_change_stdev( + data_r_mean, data_c_mean, data_r_stdev, data_c_stdev + ) + try: + item.append(round(delta)) + except ValueError: + item.append(delta) + try: + item.append(round(d_stdev)) + except ValueError: + item.append(d_stdev) + if rca_data: + rca_nr = rca_data.get(item[0], u"-") + item.insert(0, f"[{rca_nr}]" if rca_nr != u"-" else u"-") + if (len(item) == len(header)) and (item[-4] != u"Not tested"): tbl_lst.append(item) tbl_lst = _tpc_sort_table(tbl_lst) @@ -852,24 +876,16 @@ def table_perf_comparison(table, input_data): with open(csv_file, u"wt") as file_handler: file_handler.write(header_str) for test in tbl_lst: - file_handler.write(u",".join([str(item) for item in test]) + u"\n") + file_handler.write(u";".join([str(item) for item in test]) + u"\n") txt_file_name = f"{table[u'output-file']}.txt" - convert_csv_to_pretty_txt(csv_file, txt_file_name) - - if footnote: - with open(txt_file_name, u'a') as txt_file: - txt_file.writelines([ - u"\nFootnotes:\n", - u"[1] CSIT-1908 changed test methodology of dot1q tests in " - u"2-node testbeds, dot1q encapsulation is now used on both " - u"links of SUT.\n", - u" Previously dot1q was used only on a single link with the " - u"other link carrying untagged Ethernet frames. This changes " - u"results\n", - u" in slightly lower throughput in CSIT-1908 for these " - u"tests. See release notes." - ]) + convert_csv_to_pretty_txt(csv_file, txt_file_name, delimiter=u";") + + if rca_data: + footnote = rca_data.get(u"footnote", "") + if footnote: + with open(txt_file_name, u'a') as txt_file: + txt_file.writelines(footnote) # Generate html table: _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html") @@ -898,6 +914,16 @@ def table_perf_comparison_nic(table, input_data): try: header = [u"Test case", ] + rca_data = None + rca = table.get(u"rca", None) + if rca: + try: + with open(rca.get(u"data-file", ""), u"r") as rca_file: + rca_data = load(rca_file, Loader=FullLoader) + header.insert(0, rca.get(u"title", "RCA")) + except (YAMLError, IOError) as err: + logging.warning(repr(err)) + if table[u"include-tests"] == u"MRR": hdr_param = u"Rec Rate" else: @@ -917,25 +943,26 @@ def table_perf_comparison_nic(table, input_data): f"{table[u'reference'][u'title']} Stdev [Mpps]", f"{table[u'compare'][u'title']} {hdr_param} [Mpps]", f"{table[u'compare'][u'title']} Stdev [Mpps]", - u"Delta [%]" + u"Delta [%]", + u"Stdev of delta [%]" ] ) - header_str = u",".join(header) + u"\n" + header_str = u";".join(header) + u"\n" except (AttributeError, KeyError) as err: logging.error(f"The model is invalid, missing parameter: {repr(err)}") return # Prepare data to the table: tbl_dict = dict() - # topo = u"" for job, builds in table[u"reference"][u"data"].items(): - # topo = u"2n-skx" if u"2n-skx" in job else u"" for build in builds: for tst_name, tst_data in data[job][str(build)].items(): if table[u"reference"][u"nic"] not in tst_data[u"tags"]: continue tst_name_mod = _tpc_modify_test_name(tst_name) - if u"across topologies" in table[u"title"].lower(): + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: name = f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}" @@ -964,7 +991,9 @@ def table_perf_comparison_nic(table, input_data): if table[u"reference"][u"nic"] not in tst_data[u"tags"]: continue tst_name_mod = _tpc_modify_test_name(tst_name) - if u"across topologies" in table[u"title"].lower(): + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: name = \ @@ -993,7 +1022,9 @@ def table_perf_comparison_nic(table, input_data): if table[u"compare"][u"nic"] not in tst_data[u"tags"]: continue tst_name_mod = _tpc_modify_test_name(tst_name) - if u"across topologies" in table[u"title"].lower(): + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: name = f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}" @@ -1022,7 +1053,9 @@ def table_perf_comparison_nic(table, input_data): if table[u"compare"][u"nic"] not in tst_data[u"tags"]: continue tst_name_mod = _tpc_modify_test_name(tst_name) - if u"across topologies" in table[u"title"].lower(): + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: name = \ @@ -1052,7 +1085,9 @@ def table_perf_comparison_nic(table, input_data): if item[u"nic"] not in tst_data[u"tags"]: continue tst_name_mod = _tpc_modify_test_name(tst_name) - if u"across topologies" in table[u"title"].lower(): + if (u"across topologies" in table[u"title"].lower() or + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: continue @@ -1064,7 +1099,8 @@ def table_perf_comparison_nic(table, input_data): u"title"]] = list() try: if table[u"include-tests"] == u"MRR": - res = tst_data[u"result"][u"receive-rate"] + res = (tst_data[u"result"][u"receive-rate"], + tst_data[u"result"][u"receive-stdev"]) elif table[u"include-tests"] == u"PDR": res = tst_data[u"throughput"][u"PDR"][u"LOWER"] elif table[u"include-tests"] == u"NDR": @@ -1077,41 +1113,71 @@ def table_perf_comparison_nic(table, input_data): pass tbl_lst = list() - footnote = False for tst_name in tbl_dict: item = [tbl_dict[tst_name][u"name"], ] if history: if tbl_dict[tst_name].get(u"history", None) is not None: for hist_data in tbl_dict[tst_name][u"history"].values(): if hist_data: - item.append(round(mean(hist_data) / 1000000, 2)) - item.append(round(stdev(hist_data) / 1000000, 2)) + if table[u"include-tests"] == u"MRR": + item.append(round(hist_data[0][0] / 1e6, 2)) + item.append(round(hist_data[0][1] / 1e6, 2)) + else: + item.append(round(mean(hist_data) / 1e6, 2)) + item.append(round(stdev(hist_data) / 1e6, 2)) else: item.extend([u"Not tested", u"Not tested"]) else: item.extend([u"Not tested", u"Not tested"]) - data_t = tbl_dict[tst_name][u"ref-data"] - if data_t: - item.append(round(mean(data_t) / 1000000, 2)) - item.append(round(stdev(data_t) / 1000000, 2)) + data_r = tbl_dict[tst_name][u"ref-data"] + if data_r: + if table[u"include-tests"] == u"MRR": + data_r_mean = data_r[0][0] + data_r_stdev = data_r[0][1] + else: + data_r_mean = mean(data_r) + data_r_stdev = stdev(data_r) + item.append(round(data_r_mean / 1e6, 2)) + item.append(round(data_r_stdev / 1e6, 2)) else: + data_r_mean = None + data_r_stdev = None item.extend([u"Not tested", u"Not tested"]) - data_t = tbl_dict[tst_name][u"cmp-data"] - if data_t: - item.append(round(mean(data_t) / 1000000, 2)) - item.append(round(stdev(data_t) / 1000000, 2)) + data_c = tbl_dict[tst_name][u"cmp-data"] + if data_c: + if table[u"include-tests"] == u"MRR": + data_c_mean = data_c[0][0] + data_c_stdev = data_c[0][1] + else: + data_c_mean = mean(data_c) + data_c_stdev = stdev(data_c) + item.append(round(data_c_mean / 1e6, 2)) + item.append(round(data_c_stdev / 1e6, 2)) else: + data_c_mean = None + data_c_stdev = None item.extend([u"Not tested", u"Not tested"]) if item[-2] == u"Not tested": pass elif item[-4] == u"Not tested": item.append(u"New in CSIT-2001") - # elif topo == u"2n-skx" and u"dot1q" in tbl_dict[tst_name][u"name"]: - # item.append(u"See footnote [1]") - # footnote = True - elif item[-4] != 0: - item.append(int(relative_change(float(item[-4]), float(item[-2])))) - if (len(item) == len(header)) and (item[-3] != u"Not tested"): + item.append(u"New in CSIT-2001") + elif data_r_mean is not None and data_c_mean is not None: + delta, d_stdev = relative_change_stdev( + data_r_mean, data_c_mean, data_r_stdev, data_c_stdev + ) + try: + item.append(round(delta)) + except ValueError: + item.append(delta) + try: + item.append(round(d_stdev)) + except ValueError: + item.append(d_stdev) + if rca_data: + rca_nr = rca_data.get(item[0], u"-") + item.insert(0, f"[{rca_nr}]" if rca_nr != u"-" else u"-") + if (len(item) == len(header)) and (item[-4] != u"Not tested"): tbl_lst.append(item) tbl_lst = _tpc_sort_table(tbl_lst) @@ -1121,24 +1187,16 @@ def table_perf_comparison_nic(table, input_data): with open(csv_file, u"wt") as file_handler: file_handler.write(header_str) for test in tbl_lst: - file_handler.write(u",".join([str(item) for item in test]) + u"\n") + file_handler.write(u";".join([str(item) for item in test]) + u"\n") txt_file_name = f"{table[u'output-file']}.txt" - convert_csv_to_pretty_txt(csv_file, txt_file_name) - - if footnote: - with open(txt_file_name, u'a') as txt_file: - txt_file.writelines([ - u"\nFootnotes:\n", - u"[1] CSIT-1908 changed test methodology of dot1q tests in " - u"2-node testbeds, dot1q encapsulation is now used on both " - u"links of SUT.\n", - u" Previously dot1q was used only on a single link with the " - u"other link carrying untagged Ethernet frames. This changes " - u"results\n", - u" in slightly lower throughput in CSIT-1908 for these " - u"tests. See release notes." - ]) + convert_csv_to_pretty_txt(csv_file, txt_file_name, delimiter=u";") + + if rca_data: + footnote = rca_data.get(u"footnote", "") + if footnote: + with open(txt_file_name, u'a') as txt_file: + txt_file.writelines(footnote) # Generate html table: _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html") @@ -1178,7 +1236,8 @@ def table_nics_comparison(table, input_data): f"{table[u'reference'][u'title']} Stdev [Mpps]", f"{table[u'compare'][u'title']} {hdr_param} [Mpps]", f"{table[u'compare'][u'title']} Stdev [Mpps]", - u"Delta [%]" + u"Delta [%]", + u"Stdev of delta [%]" ] ) @@ -1200,9 +1259,9 @@ def table_nics_comparison(table, input_data): u"cmp-data": list() } try: - result = None if table[u"include-tests"] == u"MRR": - result = tst_data[u"result"][u"receive-rate"] + result = (tst_data[u"result"][u"receive-rate"], + tst_data[u"result"][u"receive-stdev"]) elif table[u"include-tests"] == u"PDR": result = tst_data[u"throughput"][u"PDR"][u"LOWER"] elif table[u"include-tests"] == u"NDR": @@ -1223,21 +1282,46 @@ def table_nics_comparison(table, input_data): tbl_lst = list() for tst_name in tbl_dict: item = [tbl_dict[tst_name][u"name"], ] - data_t = tbl_dict[tst_name][u"ref-data"] - if data_t: - item.append(round(mean(data_t) / 1000000, 2)) - item.append(round(stdev(data_t) / 1000000, 2)) + data_r = tbl_dict[tst_name][u"ref-data"] + if data_r: + if table[u"include-tests"] == u"MRR": + data_r_mean = data_r[0][0] + data_r_stdev = data_r[0][1] + else: + data_r_mean = mean(data_r) + data_r_stdev = stdev(data_r) + item.append(round(data_r_mean / 1e6, 2)) + item.append(round(data_r_stdev / 1e6, 2)) else: + data_r_mean = None + data_r_stdev = None item.extend([None, None]) - data_t = tbl_dict[tst_name][u"cmp-data"] - if data_t: - item.append(round(mean(data_t) / 1000000, 2)) - item.append(round(stdev(data_t) / 1000000, 2)) + data_c = tbl_dict[tst_name][u"cmp-data"] + if data_c: + if table[u"include-tests"] == u"MRR": + data_c_mean = data_c[0][0] + data_c_stdev = data_c[0][1] + else: + data_c_mean = mean(data_c) + data_c_stdev = stdev(data_c) + item.append(round(data_c_mean / 1e6, 2)) + item.append(round(data_c_stdev / 1e6, 2)) else: + data_c_mean = None + data_c_stdev = None item.extend([None, None]) - if item[-4] is not None and item[-2] is not None and item[-4] != 0: - item.append(int(relative_change(float(item[-4]), float(item[-2])))) - if len(item) == len(header): + if data_r_mean is not None and data_c_mean is not None: + delta, d_stdev = relative_change_stdev( + data_r_mean, data_c_mean, data_r_stdev, data_c_stdev + ) + try: + item.append(round(delta)) + except ValueError: + item.append(delta) + try: + item.append(round(d_stdev)) + except ValueError: + item.append(d_stdev) tbl_lst.append(item) # Sort the table according to the relative change @@ -1283,7 +1367,8 @@ def table_soak_vs_ndr(table, input_data): f"{table[u'reference'][u'title']} Stdev [Mpps]", f"{table[u'compare'][u'title']} Thput [Mpps]", f"{table[u'compare'][u'title']} Stdev [Mpps]", - u"Delta [%]", u"Stdev of delta [%]" + u"Delta [%]", + u"Stdev of delta [%]" ] header_str = u",".join(header) + u"\n" except (AttributeError, KeyError) as err: @@ -1328,7 +1413,8 @@ def table_soak_vs_ndr(table, input_data): if tst_data[u"type"] not in (u"NDRPDR", u"MRR", u"BMRR"): continue if table[u"include-tests"] == u"MRR": - result = tst_data[u"result"][u"receive-rate"] + result = (tst_data[u"result"][u"receive-rate"], + tst_data[u"result"][u"receive-stdev"]) elif table[u"include-tests"] == u"PDR": result = \ tst_data[u"throughput"][u"PDR"][u"LOWER"] @@ -1348,29 +1434,43 @@ def table_soak_vs_ndr(table, input_data): item = [tbl_dict[tst_name][u"name"], ] data_r = tbl_dict[tst_name][u"ref-data"] if data_r: - data_r_mean = mean(data_r) - item.append(round(data_r_mean / 1000000, 2)) - data_r_stdev = stdev(data_r) - item.append(round(data_r_stdev / 1000000, 2)) + if table[u"include-tests"] == u"MRR": + data_r_mean = data_r[0][0] + data_r_stdev = data_r[0][1] + else: + data_r_mean = mean(data_r) + data_r_stdev = stdev(data_r) + item.append(round(data_r_mean / 1e6, 2)) + item.append(round(data_r_stdev / 1e6, 2)) else: data_r_mean = None data_r_stdev = None item.extend([None, None]) data_c = tbl_dict[tst_name][u"cmp-data"] if data_c: - data_c_mean = mean(data_c) - item.append(round(data_c_mean / 1000000, 2)) - data_c_stdev = stdev(data_c) - item.append(round(data_c_stdev / 1000000, 2)) + if table[u"include-tests"] == u"MRR": + data_c_mean = data_c[0][0] + data_c_stdev = data_c[0][1] + else: + data_c_mean = mean(data_c) + data_c_stdev = stdev(data_c) + item.append(round(data_c_mean / 1e6, 2)) + item.append(round(data_c_stdev / 1e6, 2)) else: data_c_mean = None data_c_stdev = None item.extend([None, None]) - if data_r_mean and data_c_mean: + if data_r_mean is not None and data_c_mean is not None: delta, d_stdev = relative_change_stdev( data_r_mean, data_c_mean, data_r_stdev, data_c_stdev) - item.append(round(delta, 2)) - item.append(round(d_stdev, 2)) + try: + item.append(round(delta)) + except ValueError: + item.append(delta) + try: + item.append(round(d_stdev)) + except ValueError: + item.append(d_stdev) tbl_lst.append(item) # Sort the table according to the relative change @@ -1594,7 +1694,7 @@ def _generate_url(testbed, test_name): elif u"dnv" in testbed or u"tsh" in testbed: driver = u"ixgbe" else: - driver = u"i40e" + driver = u"dpdk" if u"acl" in test_name or \ u"macip" in test_name or \