+ for column in table[u"columns"]:
+ header.append(
+ u'"{0}"'.format(str(column[u"title"]).replace(u'"', u'""'))
+ )
+
+ for suite in suites.values:
+ # Generate data
+ suite_name = suite[u"name"]
+ table_lst = list()
+ for test in data.keys():
+ if data[test][u"parent"] not in suite_name:
+ continue
+ row_lst = list()
+ for column in table[u"columns"]:
+ try:
+ col_data = str(data[test][column[
+ u"data"].split(u" ")[1]]).replace(u'"', u'""')
+ col_data = col_data.replace(
+ u"No Data", u"Not Captured "
+ )
+ if column[u"data"].split(u" ")[1] in \
+ (u"conf-history", u"show-run"):
+ col_data = col_data.replace(u" |br| ", u"", 1)
+ col_data = f" |prein| {col_data[:-5]} |preout| "
+ row_lst.append(f'"{col_data}"')
+ except KeyError:
+ row_lst.append(u'"Not captured"')
+ table_lst.append(row_lst)
+
+ # Write the data to file
+ if table_lst:
+ file_name = (
+ f"{table[u'output-file']}_{suite_name}"
+ f"{table[u'output-file-ext']}"
+ )
+ logging.info(f" Writing file: {file_name}")
+ with open(file_name, u"w") as file_handler:
+ file_handler.write(u",".join(header) + u"\n")
+ for item in table_lst:
+ file_handler.write(u",".join(item) + u"\n")
+
+ logging.info(u" Done.")
+
+
+def _tpc_modify_test_name(test_name):
+ """Modify a test name by replacing its parts.
+
+ :param test_name: Test name to be modified.
+ :type test_name: str
+ :returns: Modified test name.
+ :rtype: str
+ """
+ test_name_mod = test_name.\
+ replace(u"-ndrpdrdisc", u""). \
+ replace(u"-ndrpdr", u"").\
+ replace(u"-pdrdisc", u""). \
+ replace(u"-ndrdisc", u"").\
+ replace(u"-pdr", u""). \
+ replace(u"-ndr", u""). \
+ replace(u"1t1c", u"1c").\
+ replace(u"2t1c", u"1c"). \
+ replace(u"2t2c", u"2c").\
+ replace(u"4t2c", u"2c"). \
+ replace(u"4t4c", u"4c").\
+ replace(u"8t4c", u"4c")
+
+ return re.sub(REGEX_NIC, u"", test_name_mod)
+
+
+def _tpc_modify_displayed_test_name(test_name):
+ """Modify a test name which is displayed in a table by replacing its parts.
+
+ :param test_name: Test name to be modified.
+ :type test_name: str
+ :returns: Modified test name.
+ :rtype: str
+ """
+ return test_name.\
+ replace(u"1t1c", u"1c").\
+ replace(u"2t1c", u"1c"). \
+ replace(u"2t2c", u"2c").\
+ replace(u"4t2c", u"2c"). \
+ replace(u"4t4c", u"4c").\
+ replace(u"8t4c", u"4c")
+
+
+def _tpc_insert_data(target, src, include_tests):
+ """Insert src data to the target structure.
+
+ :param target: Target structure where the data is placed.
+ :param src: Source data to be placed into the target stucture.
+ :param include_tests: Which results will be included (MRR, NDR, PDR).
+ :type target: list
+ :type src: dict
+ :type include_tests: str
+ """
+ try:
+ if include_tests == u"MRR":
+ target.append(src[u"result"][u"receive-rate"])
+ elif include_tests == u"PDR":
+ target.append(src[u"throughput"][u"PDR"][u"LOWER"])
+ elif include_tests == u"NDR":
+ target.append(src[u"throughput"][u"NDR"][u"LOWER"])
+ except (KeyError, TypeError):
+ pass
+
+
+def _tpc_sort_table(table):
+ """Sort the table this way:
+
+ 1. Put "New in CSIT-XXXX" at the first place.
+ 2. Put "See footnote" at the second place.
+ 3. Sort the rest by "Delta".
+
+ :param table: Table to sort.
+ :type table: list
+ :returns: Sorted table.
+ :rtype: list
+ """
+
+
+ tbl_new = list()
+ tbl_see = list()
+ tbl_delta = list()
+ for item in table:
+ if isinstance(item[-1], str):
+ if u"New in CSIT" in item[-1]:
+ tbl_new.append(item)
+ elif u"See footnote" in item[-1]:
+ tbl_see.append(item)
+ else:
+ tbl_delta.append(item)
+
+ # Sort the tables:
+ tbl_new.sort(key=lambda rel: rel[0], reverse=False)
+ tbl_see.sort(key=lambda rel: rel[0], reverse=False)
+ tbl_see.sort(key=lambda rel: rel[-1], reverse=False)
+ tbl_delta.sort(key=lambda rel: rel[-1], reverse=True)
+
+ # Put the tables together:
+ table = list()
+ table.extend(tbl_new)
+ table.extend(tbl_see)
+ table.extend(tbl_delta)
+
+ return table
+
+
+def _tpc_generate_html_table(header, data, output_file_name):
+ """Generate html table from input data with simple sorting possibility.
+
+ :param header: Table header.
+ :param data: Input data to be included in the table. It is a list of lists.
+ Inner lists are rows in the table. All inner lists must be of the same
+ length. The length of these lists must be the same as the length of the
+ header.
+ :param output_file_name: The name (relative or full path) where the
+ generated html table is written.
+ :type header: list
+ :type data: list of lists
+ :type output_file_name: str
+ """
+
+ df_data = pd.DataFrame(data, columns=header)
+
+ df_sorted = [df_data.sort_values(
+ by=[key, header[0]], ascending=[True, True]
+ if key != header[0] else [False, True]) for key in header]
+ df_sorted_rev = [df_data.sort_values(
+ by=[key, header[0]], ascending=[False, True]
+ if key != header[0] else [True, True]) for key in header]
+ df_sorted.extend(df_sorted_rev)
+
+ fill_color = [[u"#d4e4f7" if idx % 2 else u"#e9f1fb"
+ for idx in range(len(df_data))]]
+ table_header = dict(
+ values=[f"<b>{item}</b>" for item in header],
+ fill_color=u"#7eade7",
+ align=[u"left", u"center"]
+ )
+
+ fig = go.Figure()
+
+ for table in df_sorted:
+ columns = [table.get(col) for col in header]
+ fig.add_trace(
+ go.Table(
+ columnwidth=[30, 10],
+ header=table_header,
+ cells=dict(
+ values=columns,
+ fill_color=fill_color,
+ align=[u"left", u"right"]
+ )
+ )
+ )
+
+ buttons = list()
+ menu_items = [f"<b>{itm}</b> (ascending)" for itm in header]
+ menu_items_rev = [f"<b>{itm}</b> (descending)" for itm in header]
+ menu_items.extend(menu_items_rev)
+ for idx, hdr in enumerate(menu_items):
+ visible = [False, ] * len(menu_items)
+ visible[idx] = True
+ buttons.append(
+ dict(
+ label=hdr.replace(u" [Mpps]", u""),
+ method=u"update",
+ args=[{u"visible": visible}],
+ )
+ )
+
+ fig.update_layout(
+ updatemenus=[
+ go.layout.Updatemenu(
+ type=u"dropdown",
+ direction=u"down",
+ x=0.03,
+ xanchor=u"left",
+ y=1.045,
+ yanchor=u"top",
+ active=len(menu_items) - 1,
+ buttons=list(buttons)
+ )
+ ],
+ annotations=[
+ go.layout.Annotation(
+ text=u"<b>Sort by:</b>",
+ x=0,
+ xref=u"paper",
+ y=1.035,
+ yref=u"paper",
+ align=u"left",
+ showarrow=False
+ )
+ ]
+ )
+
+ ploff.plot(fig, show_link=False, auto_open=False, filename=output_file_name)
+
+
+def table_perf_comparison(table, input_data):
+ """Generate the table(s) with algorithm: table_perf_comparison
+ specified in the specification file.
+
+ :param table: Table to generate.
+ :param input_data: Data to process.
+ :type table: pandas.Series
+ :type input_data: InputData
+ """
+
+ logging.info(f" Generating the table {table.get(u'title', u'')} ...")
+
+ # Transform the data
+ logging.info(
+ f" Creating the data set for the {table.get(u'type', u'')} "
+ f"{table.get(u'title', u'')}."
+ )
+ data = input_data.filter_data(table, continue_on_error=True)
+
+ # Prepare the header of the tables
+ try:
+ header = [u"Test case", ]
+
+ if table[u"include-tests"] == u"MRR":
+ hdr_param = u"Rec Rate"
+ else:
+ hdr_param = u"Thput"
+
+ history = table.get(u"history", list())
+ for item in history:
+ header.extend(
+ [
+ f"{item[u'title']} {hdr_param} [Mpps]",
+ f"{item[u'title']} Stdev [Mpps]"
+ ]
+ )
+ header.extend(
+ [
+ f"{table[u'reference'][u'title']} {hdr_param} [Mpps]",
+ f"{table[u'reference'][u'title']} Stdev [Mpps]",
+ f"{table[u'compare'][u'title']} {hdr_param} [Mpps]",
+ f"{table[u'compare'][u'title']} Stdev [Mpps]",
+ u"Delta [%]"
+ ]
+ )
+ header_str = u",".join(header) + u"\n"
+ except (AttributeError, KeyError) as err:
+ logging.error(f"The model is invalid, missing parameter: {repr(err)}")
+ return
+
+ # Prepare data to the table:
+ tbl_dict = dict()
+ topo = ""
+ for job, builds in table[u"reference"][u"data"].items():
+ topo = u"2n-skx" if u"2n-skx" in job else u""
+ for build in builds:
+ for tst_name, tst_data in data[job][str(build)].items():
+ tst_name_mod = _tpc_modify_test_name(tst_name)
+ if u"across topologies" in table[u"title"].lower():
+ tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
+ if tbl_dict.get(tst_name_mod, None) is None:
+ groups = re.search(REGEX_NIC, tst_data[u"parent"])
+ nic = groups.group(0) if groups else u""
+ name = \
+ f"{nic}-{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
+ if u"across testbeds" in table[u"title"].lower() or \
+ u"across topologies" in table[u"title"].lower():
+ name = _tpc_modify_displayed_test_name(name)
+ tbl_dict[tst_name_mod] = {
+ u"name": name,
+ u"ref-data": list(),
+ u"cmp-data": list()
+ }
+ _tpc_insert_data(target=tbl_dict[tst_name_mod][u"ref-data"],
+ src=tst_data,
+ include_tests=table[u"include-tests"])
+
+ for job, builds in table[u"compare"][u"data"].items():
+ for build in builds:
+ for tst_name, tst_data in data[job][str(build)].items():
+ tst_name_mod = _tpc_modify_test_name(tst_name)
+ if u"across topologies" in table[u"title"].lower():
+ tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
+ if tbl_dict.get(tst_name_mod, None) is None:
+ groups = re.search(REGEX_NIC, tst_data[u"parent"])
+ nic = groups.group(0) if groups else u""
+ name = \
+ f"{nic}-{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
+ if u"across testbeds" in table[u"title"].lower() or \
+ u"across topologies" in table[u"title"].lower():
+ name = _tpc_modify_displayed_test_name(name)
+ tbl_dict[tst_name_mod] = {
+ u"name": name,
+ u"ref-data": list(),
+ u"cmp-data": list()
+ }
+ _tpc_insert_data(
+ target=tbl_dict[tst_name_mod][u"cmp-data"],
+ src=tst_data,
+ include_tests=table[u"include-tests"]
+ )
+
+ replacement = table[u"compare"].get(u"data-replacement", None)
+ if replacement:
+ create_new_list = True
+ rpl_data = input_data.filter_data(
+ table, data=replacement, continue_on_error=True)
+ for job, builds in replacement.items():
+ for build in builds:
+ for tst_name, tst_data in rpl_data[job][str(build)].items():
+ tst_name_mod = _tpc_modify_test_name(tst_name)
+ if u"across topologies" in table[u"title"].lower():
+ tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
+ if tbl_dict.get(tst_name_mod, None) is None:
+ name = \
+ f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
+ if u"across testbeds" in table[u"title"].lower() or \
+ u"across topologies" in table[u"title"].lower():
+ name = _tpc_modify_displayed_test_name(name)
+ tbl_dict[tst_name_mod] = {
+ u"name": name,
+ u"ref-data": list(),
+ u"cmp-data": list()
+ }
+ if create_new_list:
+ create_new_list = False
+ tbl_dict[tst_name_mod][u"cmp-data"] = list()
+
+ _tpc_insert_data(
+ target=tbl_dict[tst_name_mod][u"cmp-data"],
+ src=tst_data,
+ include_tests=table[u"include-tests"]
+ )
+
+ for item in history:
+ for job, builds in item[u"data"].items():
+ for build in builds:
+ for tst_name, tst_data in data[job][str(build)].items():
+ tst_name_mod = _tpc_modify_test_name(tst_name)
+ if u"across topologies" in table[u"title"].lower():
+ tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
+ if tbl_dict.get(tst_name_mod, None) is None:
+ continue
+ if tbl_dict[tst_name_mod].get(u"history", None) is None:
+ tbl_dict[tst_name_mod][u"history"] = OrderedDict()
+ if tbl_dict[tst_name_mod][u"history"].\
+ get(item[u"title"], None) is None:
+ tbl_dict[tst_name_mod][u"history"][item[
+ u"title"]] = list()
+ try:
+ if table[u"include-tests"] == u"MRR":
+ res = tst_data[u"result"][u"receive-rate"]
+ elif table[u"include-tests"] == u"PDR":
+ res = tst_data[u"throughput"][u"PDR"][u"LOWER"]
+ elif table[u"include-tests"] == u"NDR":
+ res = tst_data[u"throughput"][u"NDR"][u"LOWER"]
+ else:
+ continue
+ tbl_dict[tst_name_mod][u"history"][item[u"title"]].\
+ append(res)
+ except (TypeError, KeyError):
+ pass
+
+ tbl_lst = list()
+ footnote = False
+ for tst_name in tbl_dict:
+ item = [tbl_dict[tst_name][u"name"], ]
+ if history:
+ if tbl_dict[tst_name].get(u"history", None) is not None:
+ for hist_data in tbl_dict[tst_name][u"history"].values():
+ if hist_data:
+ item.append(round(mean(hist_data) / 1000000, 2))
+ item.append(round(stdev(hist_data) / 1000000, 2))
+ else:
+ item.extend([u"Not tested", u"Not tested"])
+ else:
+ item.extend([u"Not tested", u"Not tested"])
+ data_t = tbl_dict[tst_name][u"ref-data"]
+ if data_t:
+ item.append(round(mean(data_t) / 1000000, 2))
+ item.append(round(stdev(data_t) / 1000000, 2))
+ else:
+ item.extend([u"Not tested", u"Not tested"])
+ data_t = tbl_dict[tst_name][u"cmp-data"]
+ if data_t:
+ item.append(round(mean(data_t) / 1000000, 2))
+ item.append(round(stdev(data_t) / 1000000, 2))
+ else:
+ item.extend([u"Not tested", u"Not tested"])
+ if item[-2] == u"Not tested":
+ pass
+ elif item[-4] == u"Not tested":
+ item.append(u"New in CSIT-1908")
+ elif topo == u"2n-skx" and u"dot1q" in tbl_dict[tst_name][u"name"]:
+ item.append(u"See footnote [1]")
+ footnote = True
+ elif item[-4] != 0:
+ item.append(int(relative_change(float(item[-4]), float(item[-2]))))
+ if (len(item) == len(header)) and (item[-3] != u"Not tested"):
+ tbl_lst.append(item)
+
+ tbl_lst = _tpc_sort_table(tbl_lst)
+
+ # Generate csv tables:
+ csv_file = f"{table[u'output-file']}.csv"
+ with open(csv_file, u"w") as file_handler:
+ file_handler.write(header_str)
+ for test in tbl_lst:
+ file_handler.write(u",".join([str(item) for item in test]) + u"\n")
+
+ txt_file_name = f"{table[u'output-file']}.txt"
+ convert_csv_to_pretty_txt(csv_file, txt_file_name)
+
+ if footnote:
+ with open(txt_file_name, u'a') as txt_file:
+ txt_file.writelines([
+ u"\nFootnotes:\n",
+ u"[1] CSIT-1908 changed test methodology of dot1q tests in "
+ u"2-node testbeds, dot1q encapsulation is now used on both "
+ u"links of SUT.\n",
+ u" Previously dot1q was used only on a single link with the "
+ u"other link carrying untagged Ethernet frames. This changes "
+ u"results\n",
+ u" in slightly lower throughput in CSIT-1908 for these "
+ u"tests. See release notes."
+ ])
+
+ # Generate html table:
+ _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html")
+
+
+def table_perf_comparison_nic(table, input_data):
+ """Generate the table(s) with algorithm: table_perf_comparison
+ specified in the specification file.
+
+ :param table: Table to generate.
+ :param input_data: Data to process.
+ :type table: pandas.Series
+ :type input_data: InputData
+ """
+
+ logging.info(f" Generating the table {table.get(u'title', u'')} ...")
+
+ # Transform the data
+ logging.info(
+ f" Creating the data set for the {table.get(u'type', u'')} "
+ f"{table.get(u'title', u'')}."
+ )
+ data = input_data.filter_data(table, continue_on_error=True)
+
+ # Prepare the header of the tables
+ try:
+ header = [u"Test case", ]
+
+ if table[u"include-tests"] == u"MRR":
+ hdr_param = u"Rec Rate"
+ else:
+ hdr_param = u"Thput"
+
+ history = table.get(u"history", list())
+ for item in history:
+ header.extend(
+ [
+ f"{item[u'title']} {hdr_param} [Mpps]",
+ f"{item[u'title']} Stdev [Mpps]"
+ ]
+ )
+ header.extend(
+ [
+ f"{table[u'reference'][u'title']} {hdr_param} [Mpps]",
+ f"{table[u'reference'][u'title']} Stdev [Mpps]",
+ f"{table[u'compare'][u'title']} {hdr_param} [Mpps]",
+ f"{table[u'compare'][u'title']} Stdev [Mpps]",
+ u"Delta [%]"
+ ]
+ )
+ header_str = u",".join(header) + u"\n"
+ except (AttributeError, KeyError) as err:
+ logging.error(f"The model is invalid, missing parameter: {repr(err)}")
+ return
+
+ # Prepare data to the table:
+ tbl_dict = dict()
+ topo = u""
+ for job, builds in table[u"reference"][u"data"].items():
+ topo = u"2n-skx" if u"2n-skx" in job else u""
+ for build in builds:
+ for tst_name, tst_data in data[job][str(build)].items():
+ if table[u"reference"][u"nic"] not in tst_data[u"tags"]:
+ continue
+ tst_name_mod = _tpc_modify_test_name(tst_name)
+ if u"across topologies" in table[u"title"].lower():
+ tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
+ if tbl_dict.get(tst_name_mod, None) is None:
+ name = f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
+ if u"across testbeds" in table[u"title"].lower() or \
+ u"across topologies" in table[u"title"].lower():
+ name = _tpc_modify_displayed_test_name(name)
+ tbl_dict[tst_name_mod] = {
+ u"name": name,
+ u"ref-data": list(),
+ u"cmp-data": list()
+ }
+ _tpc_insert_data(
+ target=tbl_dict[tst_name_mod][u"ref-data"],
+ src=tst_data,
+ include_tests=table[u"include-tests"]
+ )
+
+ for job, builds in table[u"compare"][u"data"].items():
+ for build in builds:
+ for tst_name, tst_data in data[job][str(build)].items():
+ if table[u"compare"][u"nic"] not in tst_data[u"tags"]:
+ continue
+ tst_name_mod = _tpc_modify_test_name(tst_name)
+ if u"across topologies" in table[u"title"].lower():
+ tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
+ if tbl_dict.get(tst_name_mod, None) is None:
+ name = f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
+ if u"across testbeds" in table[u"title"].lower() or \
+ u"across topologies" in table[u"title"].lower():
+ name = _tpc_modify_displayed_test_name(name)
+ tbl_dict[tst_name_mod] = {
+ u"name": name,
+ u"ref-data": list(),
+ u"cmp-data": list()
+ }
+ _tpc_insert_data(
+ target=tbl_dict[tst_name_mod][u"cmp-data"],
+ src=tst_data,
+ include_tests=table[u"include-tests"]
+ )
+
+ replacement = table[u"compare"].get(u"data-replacement", None)
+ if replacement:
+ create_new_list = True
+ rpl_data = input_data.filter_data(
+ table, data=replacement, continue_on_error=True)
+ for job, builds in replacement.items():
+ for build in builds:
+ for tst_name, tst_data in rpl_data[job][str(build)].items():
+ if table[u"compare"][u"nic"] not in tst_data[u"tags"]:
+ continue
+ tst_name_mod = _tpc_modify_test_name(tst_name)
+ if u"across topologies" in table[u"title"].lower():
+ tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
+ if tbl_dict.get(tst_name_mod, None) is None:
+ name = \
+ f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
+ if u"across testbeds" in table[u"title"].lower() or \
+ u"across topologies" in table[u"title"].lower():
+ name = _tpc_modify_displayed_test_name(name)
+ tbl_dict[tst_name_mod] = {
+ u"name": name,
+ u"ref-data": list(),
+ u"cmp-data": list()
+ }
+ if create_new_list:
+ create_new_list = False
+ tbl_dict[tst_name_mod][u"cmp-data"] = list()
+
+ _tpc_insert_data(
+ target=tbl_dict[tst_name_mod][u"cmp-data"],
+ src=tst_data,
+ include_tests=table[u"include-tests"]
+ )
+
+ for item in history:
+ for job, builds in item[u"data"].items():
+ for build in builds:
+ for tst_name, tst_data in data[job][str(build)].items():
+ if item[u"nic"] not in tst_data[u"tags"]:
+ continue
+ tst_name_mod = _tpc_modify_test_name(tst_name)
+ if u"across topologies" in table[u"title"].lower():
+ tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
+ if tbl_dict.get(tst_name_mod, None) is None:
+ continue
+ if tbl_dict[tst_name_mod].get(u"history", None) is None:
+ tbl_dict[tst_name_mod][u"history"] = OrderedDict()
+ if tbl_dict[tst_name_mod][u"history"].\
+ get(item[u"title"], None) is None:
+ tbl_dict[tst_name_mod][u"history"][item[
+ u"title"]] = list()
+ try:
+ if table[u"include-tests"] == u"MRR":
+ res = tst_data[u"result"][u"receive-rate"]
+ elif table[u"include-tests"] == u"PDR":
+ res = tst_data[u"throughput"][u"PDR"][u"LOWER"]
+ elif table[u"include-tests"] == u"NDR":
+ res = tst_data[u"throughput"][u"NDR"][u"LOWER"]
+ else:
+ continue
+ tbl_dict[tst_name_mod][u"history"][item[u"title"]].\
+ append(res)
+ except (TypeError, KeyError):
+ pass