X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=resources%2Ftools%2Fpresentation%2Fgenerator_tables.py;h=5cd355307496f4fe3a5cce6523be47b3c38f9c78;hb=refs%2Fchanges%2F79%2F26079%2F1;hp=9e7ada640f6aab4d748232209d366ebcc64e4320;hpb=2d6410d275210acd535aae6873221ab0759e5d1e;p=csit.git diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py index 9e7ada640f..5cd3553074 100644 --- a/resources/tools/presentation/generator_tables.py +++ b/resources/tools/presentation/generator_tables.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019 Cisco and/or its affiliates. +# Copyright (c) 2020 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -30,7 +30,7 @@ import pandas as pd from numpy import nan, isnan -from pal_utils import mean, stdev, relative_change, classify_anomalies, \ +from pal_utils import mean, stdev, classify_anomalies, \ convert_csv_to_pretty_txt, relative_change_stdev @@ -98,10 +98,10 @@ def table_oper_data_html(table, input_data): data = input_data.merge_data(data) sort_tests = table.get(u"sort", None) - if sort_tests and sort_tests in (u"ascending", u"descending"): + if sort_tests: args = dict( inplace=True, - ascending=True if sort_tests == u"ascending" else False + ascending=(sort_tests == u"ascending") ) data.sort_index(**args) @@ -151,6 +151,17 @@ def table_oper_data_html(table, input_data): trow, u"td", attrib=dict(align=u"left", colspan=u"6") ) tcol.text = u"No Data" + + trow = ET.SubElement( + tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]) + ) + thead = ET.SubElement( + trow, u"th", attrib=dict(align=u"left", colspan=u"6") + ) + font = ET.SubElement( + thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff") + ) + font.text = u"." return str(ET.tostring(tbl, encoding=u"unicode")) tbl_hdr = ( @@ -162,7 +173,7 @@ def table_oper_data_html(table, input_data): u"Average Vector Size" ) - for dut_name, dut_data in tst_data[u"show-run"].items(): + for dut_data in tst_data[u"show-run"].values(): trow = ET.SubElement( tbl, u"tr", attrib=dict(bgcolor=colors[u"header"]) ) @@ -172,15 +183,7 @@ def table_oper_data_html(table, input_data): if dut_data.get(u"threads", None) is None: tcol.text = u"No Data" continue - bold = ET.SubElement(tcol, u"b") - bold.text = dut_name - trow = ET.SubElement( - tbl, u"tr", attrib=dict(bgcolor=colors[u"body"][0]) - ) - tcol = ET.SubElement( - trow, u"td", attrib=dict(align=u"left", colspan=u"6") - ) bold = ET.SubElement(tcol, u"b") bold.text = ( f"Host IP: {dut_data.get(u'host', '')}, " @@ -261,7 +264,7 @@ def table_oper_data_html(table, input_data): if not html_table: continue try: - file_name = f"{table[u'output-file']}_{suite[u'name']}.rst" + file_name = f"{table[u'output-file']}{suite[u'name']}.rst" with open(f"{file_name}", u'w') as html_file: logging.info(f" Writing file: {file_name}") html_file.write(u".. raw:: html\n\n\t") @@ -284,6 +287,7 @@ def table_merged_details(table, input_data): """ logging.info(f" Generating the table {table.get(u'title', u'')} ...") + # Transform the data logging.info( f" Creating the data set for the {table.get(u'type', u'')} " @@ -293,10 +297,10 @@ def table_merged_details(table, input_data): data = input_data.merge_data(data) sort_tests = table.get(u"sort", None) - if sort_tests and sort_tests in (u"ascending", u"descending"): + if sort_tests: args = dict( inplace=True, - ascending=True if sort_tests == u"ascending" else False + ascending=(sort_tests == u"ascending") ) data.sort_index(**args) @@ -323,6 +327,9 @@ def table_merged_details(table, input_data): try: col_data = str(data[test][column[ u"data"].split(u" ")[1]]).replace(u'"', u'""') + # Do not include tests with "Test Failed" in test message + if u"Test Failed" in col_data: + continue col_data = col_data.replace( u"No Data", u"Not Captured " ) @@ -335,19 +342,27 @@ def table_merged_details(table, input_data): f"{u'-'.join(col_data_lst[half:])}" col_data = f" |prein| {col_data} |preout| " elif column[u"data"].split(u" ")[1] in (u"msg", ): + # Temporary solution: remove NDR results from message: + if bool(table.get(u'remove-ndr', False)): + try: + col_data = col_data.split(u" |br| ", 1)[1] + except IndexError: + pass col_data = f" |prein| {col_data} |preout| " elif column[u"data"].split(u" ")[1] in \ - (u"conf-history", u"show-run"): + (u"conf-history", u"show-run"): col_data = col_data.replace(u" |br| ", u"", 1) col_data = f" |prein| {col_data[:-5]} |preout| " row_lst.append(f'"{col_data}"') except KeyError: row_lst.append(u'"Not captured"') - table_lst.append(row_lst) + if len(row_lst) == len(table[u"columns"]): + table_lst.append(row_lst) # Write the data to file if table_lst: - file_name = f"{table[u'output-file']}_{suite_name}.csv" + separator = u"" if table[u'output-file'].endswith(u"/") else u"_" + file_name = f"{table[u'output-file']}{separator}{suite_name}.csv" logging.info(f" Writing file: {file_name}") with open(file_name, u"wt") as file_handler: file_handler.write(u",".join(header) + u"\n") @@ -433,7 +448,6 @@ def _tpc_sort_table(table): :rtype: list """ - tbl_new = list() tbl_see = list() tbl_delta = list() @@ -449,12 +463,14 @@ def _tpc_sort_table(table): # Sort the tables: tbl_new.sort(key=lambda rel: rel[0], reverse=False) tbl_see.sort(key=lambda rel: rel[0], reverse=False) - tbl_see.sort(key=lambda rel: rel[-1], reverse=False) - tbl_delta.sort(key=lambda rel: rel[-1], reverse=True) + tbl_see.sort(key=lambda rel: rel[-2], reverse=False) + tbl_delta.sort(key=lambda rel: rel[0], reverse=False) + tbl_delta.sort(key=lambda rel: rel[-2], reverse=True) # Put the tables together: table = list() - table.extend(tbl_new) + # We do not want "New in CSIT": + # table.extend(tbl_new) table.extend(tbl_see) table.extend(tbl_delta) @@ -534,7 +550,7 @@ def _tpc_generate_html_table(header, data, output_file_name): xanchor=u"left", y=1.045, yanchor=u"top", - active=len(menu_items) - 1, + active=len(menu_items) - 2, buttons=list(buttons) ) ], @@ -596,7 +612,8 @@ def table_perf_comparison(table, input_data): f"{table[u'reference'][u'title']} Stdev [Mpps]", f"{table[u'compare'][u'title']} {hdr_param} [Mpps]", f"{table[u'compare'][u'title']} Stdev [Mpps]", - u"Delta [%]" + u"Delta [%]", + u"Stdev of delta [%]" ] ) header_str = u",".join(header) + u"\n" @@ -613,8 +630,8 @@ def table_perf_comparison(table, input_data): for tst_name, tst_data in data[job][str(build)].items(): tst_name_mod = _tpc_modify_test_name(tst_name) if (u"across topologies" in table[u"title"].lower() or - (u" 3n-" in table[u"title"].lower() and - u" 2n-" in table[u"title"].lower())): + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: groups = re.search(REGEX_NIC, tst_data[u"parent"]) @@ -643,8 +660,8 @@ def table_perf_comparison(table, input_data): for tst_name, tst_data in rpl_data[job][str(build)].items(): tst_name_mod = _tpc_modify_test_name(tst_name) if (u"across topologies" in table[u"title"].lower() or - (u" 3n-" in table[u"title"].lower() and - u" 2n-" in table[u"title"].lower())): + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: name = \ @@ -672,8 +689,8 @@ def table_perf_comparison(table, input_data): for tst_name, tst_data in data[job][str(build)].items(): tst_name_mod = _tpc_modify_test_name(tst_name) if (u"across topologies" in table[u"title"].lower() or - (u" 3n-" in table[u"title"].lower() and - u" 2n-" in table[u"title"].lower())): + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: groups = re.search(REGEX_NIC, tst_data[u"parent"]) @@ -704,8 +721,8 @@ def table_perf_comparison(table, input_data): for tst_name, tst_data in rpl_data[job][str(build)].items(): tst_name_mod = _tpc_modify_test_name(tst_name) if (u"across topologies" in table[u"title"].lower() or - (u" 3n-" in table[u"title"].lower() and - u" 2n-" in table[u"title"].lower())): + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: name = \ @@ -734,8 +751,8 @@ def table_perf_comparison(table, input_data): for tst_name, tst_data in data[job][str(build)].items(): tst_name_mod = _tpc_modify_test_name(tst_name) if (u"across topologies" in table[u"title"].lower() or - (u" 3n-" in table[u"title"].lower() and - u" 2n-" in table[u"title"].lower())): + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: continue @@ -773,28 +790,47 @@ def table_perf_comparison(table, input_data): item.extend([u"Not tested", u"Not tested"]) else: item.extend([u"Not tested", u"Not tested"]) - data_t = tbl_dict[tst_name][u"ref-data"] - if data_t: - item.append(round(mean(data_t) / 1000000, 2)) - item.append(round(stdev(data_t) / 1000000, 2)) + data_r = tbl_dict[tst_name][u"ref-data"] + if data_r: + data_r_mean = mean(data_r) + item.append(round(data_r_mean / 1000000, 2)) + data_r_stdev = stdev(data_r) + item.append(round(data_r_stdev / 1000000, 2)) else: + data_r_mean = None + data_r_stdev = None item.extend([u"Not tested", u"Not tested"]) - data_t = tbl_dict[tst_name][u"cmp-data"] - if data_t: - item.append(round(mean(data_t) / 1000000, 2)) - item.append(round(stdev(data_t) / 1000000, 2)) + data_c = tbl_dict[tst_name][u"cmp-data"] + if data_c: + data_c_mean = mean(data_c) + item.append(round(data_c_mean / 1000000, 2)) + data_c_stdev = stdev(data_c) + item.append(round(data_c_stdev / 1000000, 2)) else: + data_c_mean = None + data_c_stdev = None item.extend([u"Not tested", u"Not tested"]) if item[-2] == u"Not tested": pass elif item[-4] == u"Not tested": item.append(u"New in CSIT-2001") + item.append(u"New in CSIT-2001") # elif topo == u"2n-skx" and u"dot1q" in tbl_dict[tst_name][u"name"]: # item.append(u"See footnote [1]") # footnote = True - elif item[-4] != 0: - item.append(int(relative_change(float(item[-4]), float(item[-2])))) - if (len(item) == len(header)) and (item[-3] != u"Not tested"): + elif data_r_mean and data_c_mean: + delta, d_stdev = relative_change_stdev( + data_r_mean, data_c_mean, data_r_stdev, data_c_stdev + ) + try: + item.append(round(delta)) + except ValueError: + item.append(delta) + try: + item.append(round(d_stdev)) + except ValueError: + item.append(d_stdev) + if (len(item) == len(header)) and (item[-4] != u"Not tested"): tbl_lst.append(item) tbl_lst = _tpc_sort_table(tbl_lst) @@ -869,7 +905,8 @@ def table_perf_comparison_nic(table, input_data): f"{table[u'reference'][u'title']} Stdev [Mpps]", f"{table[u'compare'][u'title']} {hdr_param} [Mpps]", f"{table[u'compare'][u'title']} Stdev [Mpps]", - u"Delta [%]" + u"Delta [%]", + u"Stdev of delta [%]" ] ) header_str = u",".join(header) + u"\n" @@ -888,8 +925,8 @@ def table_perf_comparison_nic(table, input_data): continue tst_name_mod = _tpc_modify_test_name(tst_name) if (u"across topologies" in table[u"title"].lower() or - (u" 3n-" in table[u"title"].lower() and - u" 2n-" in table[u"title"].lower())): + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: name = f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}" @@ -919,8 +956,8 @@ def table_perf_comparison_nic(table, input_data): continue tst_name_mod = _tpc_modify_test_name(tst_name) if (u"across topologies" in table[u"title"].lower() or - (u" 3n-" in table[u"title"].lower() and - u" 2n-" in table[u"title"].lower())): + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: name = \ @@ -950,8 +987,8 @@ def table_perf_comparison_nic(table, input_data): continue tst_name_mod = _tpc_modify_test_name(tst_name) if (u"across topologies" in table[u"title"].lower() or - (u" 3n-" in table[u"title"].lower() and - u" 2n-" in table[u"title"].lower())): + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: name = f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}" @@ -981,8 +1018,8 @@ def table_perf_comparison_nic(table, input_data): continue tst_name_mod = _tpc_modify_test_name(tst_name) if (u"across topologies" in table[u"title"].lower() or - (u" 3n-" in table[u"title"].lower() and - u" 2n-" in table[u"title"].lower())): + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: name = \ @@ -1013,8 +1050,8 @@ def table_perf_comparison_nic(table, input_data): continue tst_name_mod = _tpc_modify_test_name(tst_name) if (u"across topologies" in table[u"title"].lower() or - (u" 3n-" in table[u"title"].lower() and - u" 2n-" in table[u"title"].lower())): + (u" 3n-" in table[u"title"].lower() and + u" 2n-" in table[u"title"].lower())): tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") if tbl_dict.get(tst_name_mod, None) is None: continue @@ -1052,28 +1089,47 @@ def table_perf_comparison_nic(table, input_data): item.extend([u"Not tested", u"Not tested"]) else: item.extend([u"Not tested", u"Not tested"]) - data_t = tbl_dict[tst_name][u"ref-data"] - if data_t: - item.append(round(mean(data_t) / 1000000, 2)) - item.append(round(stdev(data_t) / 1000000, 2)) + data_r = tbl_dict[tst_name][u"ref-data"] + if data_r: + data_r_mean = mean(data_r) + item.append(round(data_r_mean / 1000000, 2)) + data_r_stdev = stdev(data_r) + item.append(round(data_r_stdev / 1000000, 2)) else: + data_r_mean = None + data_r_stdev = None item.extend([u"Not tested", u"Not tested"]) - data_t = tbl_dict[tst_name][u"cmp-data"] - if data_t: - item.append(round(mean(data_t) / 1000000, 2)) - item.append(round(stdev(data_t) / 1000000, 2)) + data_c = tbl_dict[tst_name][u"cmp-data"] + if data_c: + data_c_mean = mean(data_c) + item.append(round(data_c_mean / 1000000, 2)) + data_c_stdev = stdev(data_c) + item.append(round(data_c_stdev / 1000000, 2)) else: + data_c_mean = None + data_c_stdev = None item.extend([u"Not tested", u"Not tested"]) if item[-2] == u"Not tested": pass elif item[-4] == u"Not tested": item.append(u"New in CSIT-2001") + item.append(u"New in CSIT-2001") # elif topo == u"2n-skx" and u"dot1q" in tbl_dict[tst_name][u"name"]: # item.append(u"See footnote [1]") # footnote = True - elif item[-4] != 0: - item.append(int(relative_change(float(item[-4]), float(item[-2])))) - if (len(item) == len(header)) and (item[-3] != u"Not tested"): + elif data_r_mean and data_c_mean: + delta, d_stdev = relative_change_stdev( + data_r_mean, data_c_mean, data_r_stdev, data_c_stdev + ) + try: + item.append(round(delta)) + except ValueError: + item.append(delta) + try: + item.append(round(d_stdev)) + except ValueError: + item.append(d_stdev) + if (len(item) == len(header)) and (item[-4] != u"Not tested"): tbl_lst.append(item) tbl_lst = _tpc_sort_table(tbl_lst) @@ -1140,7 +1196,8 @@ def table_nics_comparison(table, input_data): f"{table[u'reference'][u'title']} Stdev [Mpps]", f"{table[u'compare'][u'title']} {hdr_param} [Mpps]", f"{table[u'compare'][u'title']} Stdev [Mpps]", - u"Delta [%]" + u"Delta [%]", + u"Stdev of delta [%]" ] ) @@ -1162,7 +1219,6 @@ def table_nics_comparison(table, input_data): u"cmp-data": list() } try: - result = None if table[u"include-tests"] == u"MRR": result = tst_data[u"result"][u"receive-rate"] elif table[u"include-tests"] == u"PDR": @@ -1185,21 +1241,38 @@ def table_nics_comparison(table, input_data): tbl_lst = list() for tst_name in tbl_dict: item = [tbl_dict[tst_name][u"name"], ] - data_t = tbl_dict[tst_name][u"ref-data"] - if data_t: - item.append(round(mean(data_t) / 1000000, 2)) - item.append(round(stdev(data_t) / 1000000, 2)) + data_r = tbl_dict[tst_name][u"ref-data"] + if data_r: + data_r_mean = mean(data_r) + item.append(round(data_r_mean / 1000000, 2)) + data_r_stdev = stdev(data_r) + item.append(round(data_r_stdev / 1000000, 2)) else: + data_r_mean = None + data_r_stdev = None item.extend([None, None]) - data_t = tbl_dict[tst_name][u"cmp-data"] - if data_t: - item.append(round(mean(data_t) / 1000000, 2)) - item.append(round(stdev(data_t) / 1000000, 2)) + data_c = tbl_dict[tst_name][u"cmp-data"] + if data_c: + data_c_mean = mean(data_c) + item.append(round(data_c_mean / 1000000, 2)) + data_c_stdev = stdev(data_c) + item.append(round(data_c_stdev / 1000000, 2)) else: + data_c_mean = None + data_c_stdev = None item.extend([None, None]) - if item[-4] is not None and item[-2] is not None and item[-4] != 0: - item.append(int(relative_change(float(item[-4]), float(item[-2])))) - if len(item) == len(header): + if data_r_mean and data_c_mean: + delta, d_stdev = relative_change_stdev( + data_r_mean, data_c_mean, data_r_stdev, data_c_stdev + ) + try: + item.append(round(delta)) + except ValueError: + item.append(delta) + try: + item.append(round(d_stdev)) + except ValueError: + item.append(d_stdev) tbl_lst.append(item) # Sort the table according to the relative change @@ -1245,7 +1318,8 @@ def table_soak_vs_ndr(table, input_data): f"{table[u'reference'][u'title']} Stdev [Mpps]", f"{table[u'compare'][u'title']} Thput [Mpps]", f"{table[u'compare'][u'title']} Stdev [Mpps]", - u"Delta [%]", u"Stdev of delta [%]" + u"Delta [%]", + u"Stdev of delta [%]" ] header_str = u",".join(header) + u"\n" except (AttributeError, KeyError) as err: @@ -1331,8 +1405,14 @@ def table_soak_vs_ndr(table, input_data): if data_r_mean and data_c_mean: delta, d_stdev = relative_change_stdev( data_r_mean, data_c_mean, data_r_stdev, data_c_stdev) - item.append(round(delta, 2)) - item.append(round(d_stdev, 2)) + try: + item.append(round(delta)) + except ValueError: + item.append(delta) + try: + item.append(round(d_stdev)) + except ValueError: + item.append(d_stdev) tbl_lst.append(item) # Sort the table according to the relative change