- :type data: list of lists
- :type output_file_name: str
- """
-
- df_data = pd.DataFrame(data, columns=header)
-
- df_sorted = [df_data.sort_values(
- by=[key, header[0]], ascending=[True, True]
- if key != header[0] else [False, True]) for key in header]
- df_sorted_rev = [df_data.sort_values(
- by=[key, header[0]], ascending=[False, True]
- if key != header[0] else [True, True]) for key in header]
- df_sorted.extend(df_sorted_rev)
-
- fill_color = [[u"#d4e4f7" if idx % 2 else u"#e9f1fb"
- for idx in range(len(df_data))]]
- table_header = dict(
- values=[f"<b>{item}</b>" for item in header],
- fill_color=u"#7eade7",
- align=[u"left", u"center"]
- )
-
- fig = go.Figure()
-
- for table in df_sorted:
- columns = [table.get(col) for col in header]
- fig.add_trace(
- go.Table(
- columnwidth=[30, 10],
- header=table_header,
- cells=dict(
- values=columns,
- fill_color=fill_color,
- align=[u"left", u"right"]
- )
- )
- )
-
- buttons = list()
- menu_items = [f"<b>{itm}</b> (ascending)" for itm in header]
- menu_items_rev = [f"<b>{itm}</b> (descending)" for itm in header]
- menu_items.extend(menu_items_rev)
- for idx, hdr in enumerate(menu_items):
- visible = [False, ] * len(menu_items)
- visible[idx] = True
- buttons.append(
- dict(
- label=hdr.replace(u" [Mpps]", u""),
- method=u"update",
- args=[{u"visible": visible}],
- )
- )
-
- fig.update_layout(
- updatemenus=[
- go.layout.Updatemenu(
- type=u"dropdown",
- direction=u"down",
- x=0.03,
- xanchor=u"left",
- y=1.045,
- yanchor=u"top",
- active=len(menu_items) - 1,
- buttons=list(buttons)
- )
- ],
- annotations=[
- go.layout.Annotation(
- text=u"<b>Sort by:</b>",
- x=0,
- xref=u"paper",
- y=1.035,
- yref=u"paper",
- align=u"left",
- showarrow=False
- )
- ]
- )
-
- ploff.plot(fig, show_link=False, auto_open=False, filename=output_file_name)
-
-
-def table_perf_comparison(table, input_data):
- """Generate the table(s) with algorithm: table_perf_comparison
- specified in the specification file.
-
- :param table: Table to generate.
- :param input_data: Data to process.
- :type table: pandas.Series
- :type input_data: InputData
- """
-
- logging.info(f" Generating the table {table.get(u'title', u'')} ...")
-
- # Transform the data
- logging.info(
- f" Creating the data set for the {table.get(u'type', u'')} "
- f"{table.get(u'title', u'')}."
- )
- data = input_data.filter_data(table, continue_on_error=True)
-
- # Prepare the header of the tables
- try:
- header = [u"Test case", ]
-
- if table[u"include-tests"] == u"MRR":
- hdr_param = u"Rec Rate"
- else:
- hdr_param = u"Thput"
-
- history = table.get(u"history", list())
- for item in history:
- header.extend(
- [
- f"{item[u'title']} {hdr_param} [Mpps]",
- f"{item[u'title']} Stdev [Mpps]"
- ]
- )
- header.extend(
- [
- f"{table[u'reference'][u'title']} {hdr_param} [Mpps]",
- f"{table[u'reference'][u'title']} Stdev [Mpps]",
- f"{table[u'compare'][u'title']} {hdr_param} [Mpps]",
- f"{table[u'compare'][u'title']} Stdev [Mpps]",
- u"Delta [%]"
- ]
- )
- header_str = u",".join(header) + u"\n"
- except (AttributeError, KeyError) as err:
- logging.error(f"The model is invalid, missing parameter: {repr(err)}")
- return
-
- # Prepare data to the table:
- tbl_dict = dict()
- # topo = ""
- for job, builds in table[u"reference"][u"data"].items():
- # topo = u"2n-skx" if u"2n-skx" in job else u""
- for build in builds:
- for tst_name, tst_data in data[job][str(build)].items():
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if (u"across topologies" in table[u"title"].lower() or
- (u" 3n-" in table[u"title"].lower() and
- u" 2n-" in table[u"title"].lower())):
- tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
- if tbl_dict.get(tst_name_mod, None) is None:
- groups = re.search(REGEX_NIC, tst_data[u"parent"])
- nic = groups.group(0) if groups else u""
- name = \
- f"{nic}-{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
- if u"across testbeds" in table[u"title"].lower() or \
- u"across topologies" in table[u"title"].lower():
- name = _tpc_modify_displayed_test_name(name)
- tbl_dict[tst_name_mod] = {
- u"name": name,
- u"ref-data": list(),
- u"cmp-data": list()
- }
- _tpc_insert_data(target=tbl_dict[tst_name_mod][u"ref-data"],
- src=tst_data,
- include_tests=table[u"include-tests"])
-
- replacement = table[u"reference"].get(u"data-replacement", None)
- if replacement:
- create_new_list = True
- rpl_data = input_data.filter_data(
- table, data=replacement, continue_on_error=True)
- for job, builds in replacement.items():
- for build in builds:
- for tst_name, tst_data in rpl_data[job][str(build)].items():
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if (u"across topologies" in table[u"title"].lower() or
- (u" 3n-" in table[u"title"].lower() and
- u" 2n-" in table[u"title"].lower())):
- tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
- if tbl_dict.get(tst_name_mod, None) is None:
- name = \
- f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
- if u"across testbeds" in table[u"title"].lower() or \
- u"across topologies" in table[u"title"].lower():
- name = _tpc_modify_displayed_test_name(name)
- tbl_dict[tst_name_mod] = {
- u"name": name,
- u"ref-data": list(),
- u"cmp-data": list()
- }
- if create_new_list:
- create_new_list = False
- tbl_dict[tst_name_mod][u"ref-data"] = list()
-
- _tpc_insert_data(
- target=tbl_dict[tst_name_mod][u"ref-data"],
- src=tst_data,
- include_tests=table[u"include-tests"]
- )
-
- for job, builds in table[u"compare"][u"data"].items():
- for build in builds:
- for tst_name, tst_data in data[job][str(build)].items():
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if (u"across topologies" in table[u"title"].lower() or
- (u" 3n-" in table[u"title"].lower() and
- u" 2n-" in table[u"title"].lower())):
- tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
- if tbl_dict.get(tst_name_mod, None) is None:
- groups = re.search(REGEX_NIC, tst_data[u"parent"])
- nic = groups.group(0) if groups else u""
- name = \
- f"{nic}-{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
- if u"across testbeds" in table[u"title"].lower() or \
- u"across topologies" in table[u"title"].lower():
- name = _tpc_modify_displayed_test_name(name)
- tbl_dict[tst_name_mod] = {
- u"name": name,
- u"ref-data": list(),
- u"cmp-data": list()
- }
- _tpc_insert_data(
- target=tbl_dict[tst_name_mod][u"cmp-data"],
- src=tst_data,
- include_tests=table[u"include-tests"]
- )
-
- replacement = table[u"compare"].get(u"data-replacement", None)
- if replacement:
- create_new_list = True
- rpl_data = input_data.filter_data(
- table, data=replacement, continue_on_error=True)
- for job, builds in replacement.items():
- for build in builds:
- for tst_name, tst_data in rpl_data[job][str(build)].items():
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if (u"across topologies" in table[u"title"].lower() or
- (u" 3n-" in table[u"title"].lower() and
- u" 2n-" in table[u"title"].lower())):
- tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
- if tbl_dict.get(tst_name_mod, None) is None:
- name = \
- f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
- if u"across testbeds" in table[u"title"].lower() or \
- u"across topologies" in table[u"title"].lower():
- name = _tpc_modify_displayed_test_name(name)
- tbl_dict[tst_name_mod] = {
- u"name": name,
- u"ref-data": list(),
- u"cmp-data": list()
- }
- if create_new_list:
- create_new_list = False
- tbl_dict[tst_name_mod][u"cmp-data"] = list()
-
- _tpc_insert_data(
- target=tbl_dict[tst_name_mod][u"cmp-data"],
- src=tst_data,
- include_tests=table[u"include-tests"]
- )
-
- for item in history:
- for job, builds in item[u"data"].items():
- for build in builds:
- for tst_name, tst_data in data[job][str(build)].items():
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if (u"across topologies" in table[u"title"].lower() or
- (u" 3n-" in table[u"title"].lower() and
- u" 2n-" in table[u"title"].lower())):
- tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
- if tbl_dict.get(tst_name_mod, None) is None:
- continue
- if tbl_dict[tst_name_mod].get(u"history", None) is None:
- tbl_dict[tst_name_mod][u"history"] = OrderedDict()
- if tbl_dict[tst_name_mod][u"history"].\
- get(item[u"title"], None) is None:
- tbl_dict[tst_name_mod][u"history"][item[
- u"title"]] = list()
- try:
- if table[u"include-tests"] == u"MRR":
- res = tst_data[u"result"][u"receive-rate"]
- elif table[u"include-tests"] == u"PDR":
- res = tst_data[u"throughput"][u"PDR"][u"LOWER"]
- elif table[u"include-tests"] == u"NDR":
- res = tst_data[u"throughput"][u"NDR"][u"LOWER"]
- else:
- continue
- tbl_dict[tst_name_mod][u"history"][item[u"title"]].\
- append(res)
- except (TypeError, KeyError):
- pass
-
- tbl_lst = list()
- footnote = False
- for tst_name in tbl_dict:
- item = [tbl_dict[tst_name][u"name"], ]
- if history:
- if tbl_dict[tst_name].get(u"history", None) is not None:
- for hist_data in tbl_dict[tst_name][u"history"].values():
- if hist_data:
- item.append(round(mean(hist_data) / 1000000, 2))
- item.append(round(stdev(hist_data) / 1000000, 2))
- else:
- item.extend([u"Not tested", u"Not tested"])
- else:
- item.extend([u"Not tested", u"Not tested"])
- data_t = tbl_dict[tst_name][u"ref-data"]
- if data_t:
- item.append(round(mean(data_t) / 1000000, 2))
- item.append(round(stdev(data_t) / 1000000, 2))
- else:
- item.extend([u"Not tested", u"Not tested"])
- data_t = tbl_dict[tst_name][u"cmp-data"]
- if data_t:
- item.append(round(mean(data_t) / 1000000, 2))
- item.append(round(stdev(data_t) / 1000000, 2))
- else:
- item.extend([u"Not tested", u"Not tested"])
- if item[-2] == u"Not tested":
- pass
- elif item[-4] == u"Not tested":
- item.append(u"New in CSIT-2001")
- # elif topo == u"2n-skx" and u"dot1q" in tbl_dict[tst_name][u"name"]:
- # item.append(u"See footnote [1]")
- # footnote = True
- elif item[-4] != 0:
- item.append(int(relative_change(float(item[-4]), float(item[-2]))))
- if (len(item) == len(header)) and (item[-3] != u"Not tested"):
- tbl_lst.append(item)
-
- tbl_lst = _tpc_sort_table(tbl_lst)
-
- # Generate csv tables:
- csv_file = f"{table[u'output-file']}.csv"
- with open(csv_file, u"wt") as file_handler:
- file_handler.write(header_str)
- for test in tbl_lst:
- file_handler.write(u",".join([str(item) for item in test]) + u"\n")
-
- txt_file_name = f"{table[u'output-file']}.txt"
- convert_csv_to_pretty_txt(csv_file, txt_file_name)
-
- if footnote:
- with open(txt_file_name, u'a') as txt_file:
- txt_file.writelines([
- u"\nFootnotes:\n",
- u"[1] CSIT-1908 changed test methodology of dot1q tests in "
- u"2-node testbeds, dot1q encapsulation is now used on both "
- u"links of SUT.\n",
- u" Previously dot1q was used only on a single link with the "
- u"other link carrying untagged Ethernet frames. This changes "
- u"results\n",
- u" in slightly lower throughput in CSIT-1908 for these "
- u"tests. See release notes."
- ])
-
- # Generate html table:
- _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html")
-
-
-def table_perf_comparison_nic(table, input_data):
- """Generate the table(s) with algorithm: table_perf_comparison
- specified in the specification file.
-
- :param table: Table to generate.
- :param input_data: Data to process.
- :type table: pandas.Series
- :type input_data: InputData
- """
-
- logging.info(f" Generating the table {table.get(u'title', u'')} ...")
-
- # Transform the data
- logging.info(
- f" Creating the data set for the {table.get(u'type', u'')} "
- f"{table.get(u'title', u'')}."
- )
- data = input_data.filter_data(table, continue_on_error=True)
-
- # Prepare the header of the tables
- try:
- header = [u"Test case", ]
-
- if table[u"include-tests"] == u"MRR":
- hdr_param = u"Rec Rate"
- else:
- hdr_param = u"Thput"
-
- history = table.get(u"history", list())
- for item in history:
- header.extend(
- [
- f"{item[u'title']} {hdr_param} [Mpps]",
- f"{item[u'title']} Stdev [Mpps]"
- ]
- )
- header.extend(
- [
- f"{table[u'reference'][u'title']} {hdr_param} [Mpps]",
- f"{table[u'reference'][u'title']} Stdev [Mpps]",
- f"{table[u'compare'][u'title']} {hdr_param} [Mpps]",
- f"{table[u'compare'][u'title']} Stdev [Mpps]",
- u"Delta [%]"
- ]
- )
- header_str = u",".join(header) + u"\n"
- except (AttributeError, KeyError) as err:
- logging.error(f"The model is invalid, missing parameter: {repr(err)}")
- return
-
- # Prepare data to the table:
- tbl_dict = dict()
- # topo = u""
- for job, builds in table[u"reference"][u"data"].items():
- # topo = u"2n-skx" if u"2n-skx" in job else u""
- for build in builds:
- for tst_name, tst_data in data[job][str(build)].items():
- if table[u"reference"][u"nic"] not in tst_data[u"tags"]:
- continue
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if (u"across topologies" in table[u"title"].lower() or
- (u" 3n-" in table[u"title"].lower() and
- u" 2n-" in table[u"title"].lower())):
- tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
- if tbl_dict.get(tst_name_mod, None) is None:
- name = f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
- if u"across testbeds" in table[u"title"].lower() or \
- u"across topologies" in table[u"title"].lower():
- name = _tpc_modify_displayed_test_name(name)
- tbl_dict[tst_name_mod] = {
- u"name": name,
- u"ref-data": list(),
- u"cmp-data": list()
- }
- _tpc_insert_data(
- target=tbl_dict[tst_name_mod][u"ref-data"],
- src=tst_data,
- include_tests=table[u"include-tests"]
- )
-
- replacement = table[u"reference"].get(u"data-replacement", None)
- if replacement:
- create_new_list = True
- rpl_data = input_data.filter_data(
- table, data=replacement, continue_on_error=True)
- for job, builds in replacement.items():
- for build in builds:
- for tst_name, tst_data in rpl_data[job][str(build)].items():
- if table[u"reference"][u"nic"] not in tst_data[u"tags"]:
- continue
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if (u"across topologies" in table[u"title"].lower() or
- (u" 3n-" in table[u"title"].lower() and
- u" 2n-" in table[u"title"].lower())):
- tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
- if tbl_dict.get(tst_name_mod, None) is None:
- name = \
- f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
- if u"across testbeds" in table[u"title"].lower() or \
- u"across topologies" in table[u"title"].lower():
- name = _tpc_modify_displayed_test_name(name)
- tbl_dict[tst_name_mod] = {
- u"name": name,
- u"ref-data": list(),
- u"cmp-data": list()
- }
- if create_new_list:
- create_new_list = False
- tbl_dict[tst_name_mod][u"ref-data"] = list()
-
- _tpc_insert_data(
- target=tbl_dict[tst_name_mod][u"ref-data"],
- src=tst_data,
- include_tests=table[u"include-tests"]
- )
-
- for job, builds in table[u"compare"][u"data"].items():
- for build in builds:
- for tst_name, tst_data in data[job][str(build)].items():
- if table[u"compare"][u"nic"] not in tst_data[u"tags"]:
- continue
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if (u"across topologies" in table[u"title"].lower() or
- (u" 3n-" in table[u"title"].lower() and
- u" 2n-" in table[u"title"].lower())):
- tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
- if tbl_dict.get(tst_name_mod, None) is None:
- name = f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
- if u"across testbeds" in table[u"title"].lower() or \
- u"across topologies" in table[u"title"].lower():
- name = _tpc_modify_displayed_test_name(name)
- tbl_dict[tst_name_mod] = {
- u"name": name,
- u"ref-data": list(),
- u"cmp-data": list()
- }
- _tpc_insert_data(
- target=tbl_dict[tst_name_mod][u"cmp-data"],
- src=tst_data,
- include_tests=table[u"include-tests"]
- )
-
- replacement = table[u"compare"].get(u"data-replacement", None)
- if replacement:
- create_new_list = True
- rpl_data = input_data.filter_data(
- table, data=replacement, continue_on_error=True)
- for job, builds in replacement.items():
- for build in builds:
- for tst_name, tst_data in rpl_data[job][str(build)].items():
- if table[u"compare"][u"nic"] not in tst_data[u"tags"]:
- continue
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if (u"across topologies" in table[u"title"].lower() or
- (u" 3n-" in table[u"title"].lower() and
- u" 2n-" in table[u"title"].lower())):
- tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
- if tbl_dict.get(tst_name_mod, None) is None:
- name = \
- f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
- if u"across testbeds" in table[u"title"].lower() or \
- u"across topologies" in table[u"title"].lower():
- name = _tpc_modify_displayed_test_name(name)
- tbl_dict[tst_name_mod] = {
- u"name": name,
- u"ref-data": list(),
- u"cmp-data": list()
- }
- if create_new_list:
- create_new_list = False
- tbl_dict[tst_name_mod][u"cmp-data"] = list()
-
- _tpc_insert_data(
- target=tbl_dict[tst_name_mod][u"cmp-data"],
- src=tst_data,
- include_tests=table[u"include-tests"]
- )
-
- for item in history:
- for job, builds in item[u"data"].items():
- for build in builds:
- for tst_name, tst_data in data[job][str(build)].items():
- if item[u"nic"] not in tst_data[u"tags"]:
- continue
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if (u"across topologies" in table[u"title"].lower() or
- (u" 3n-" in table[u"title"].lower() and
- u" 2n-" in table[u"title"].lower())):
- tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
- if tbl_dict.get(tst_name_mod, None) is None:
- continue
- if tbl_dict[tst_name_mod].get(u"history", None) is None:
- tbl_dict[tst_name_mod][u"history"] = OrderedDict()
- if tbl_dict[tst_name_mod][u"history"].\
- get(item[u"title"], None) is None:
- tbl_dict[tst_name_mod][u"history"][item[
- u"title"]] = list()
- try:
- if table[u"include-tests"] == u"MRR":
- res = tst_data[u"result"][u"receive-rate"]
- elif table[u"include-tests"] == u"PDR":
- res = tst_data[u"throughput"][u"PDR"][u"LOWER"]
- elif table[u"include-tests"] == u"NDR":
- res = tst_data[u"throughput"][u"NDR"][u"LOWER"]
- else:
- continue
- tbl_dict[tst_name_mod][u"history"][item[u"title"]].\
- append(res)
- except (TypeError, KeyError):
- pass
-
- tbl_lst = list()
- footnote = False
- for tst_name in tbl_dict:
- item = [tbl_dict[tst_name][u"name"], ]
- if history:
- if tbl_dict[tst_name].get(u"history", None) is not None:
- for hist_data in tbl_dict[tst_name][u"history"].values():
- if hist_data:
- item.append(round(mean(hist_data) / 1000000, 2))
- item.append(round(stdev(hist_data) / 1000000, 2))
- else:
- item.extend([u"Not tested", u"Not tested"])
- else:
- item.extend([u"Not tested", u"Not tested"])
- data_t = tbl_dict[tst_name][u"ref-data"]
- if data_t:
- item.append(round(mean(data_t) / 1000000, 2))
- item.append(round(stdev(data_t) / 1000000, 2))
- else:
- item.extend([u"Not tested", u"Not tested"])
- data_t = tbl_dict[tst_name][u"cmp-data"]
- if data_t:
- item.append(round(mean(data_t) / 1000000, 2))
- item.append(round(stdev(data_t) / 1000000, 2))
- else:
- item.extend([u"Not tested", u"Not tested"])
- if item[-2] == u"Not tested":
- pass
- elif item[-4] == u"Not tested":
- item.append(u"New in CSIT-2001")
- # elif topo == u"2n-skx" and u"dot1q" in tbl_dict[tst_name][u"name"]:
- # item.append(u"See footnote [1]")
- # footnote = True
- elif item[-4] != 0:
- item.append(int(relative_change(float(item[-4]), float(item[-2]))))
- if (len(item) == len(header)) and (item[-3] != u"Not tested"):
- tbl_lst.append(item)
-
- tbl_lst = _tpc_sort_table(tbl_lst)
-
- # Generate csv tables:
- csv_file = f"{table[u'output-file']}.csv"
- with open(csv_file, u"wt") as file_handler:
- file_handler.write(header_str)
- for test in tbl_lst:
- file_handler.write(u",".join([str(item) for item in test]) + u"\n")
-
- txt_file_name = f"{table[u'output-file']}.txt"
- convert_csv_to_pretty_txt(csv_file, txt_file_name)
-
- if footnote:
- with open(txt_file_name, u'a') as txt_file:
- txt_file.writelines([
- u"\nFootnotes:\n",
- u"[1] CSIT-1908 changed test methodology of dot1q tests in "
- u"2-node testbeds, dot1q encapsulation is now used on both "
- u"links of SUT.\n",
- u" Previously dot1q was used only on a single link with the "
- u"other link carrying untagged Ethernet frames. This changes "
- u"results\n",
- u" in slightly lower throughput in CSIT-1908 for these "
- u"tests. See release notes."
- ])
-
- # Generate html table:
- _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html")
-
-
-def table_nics_comparison(table, input_data):
- """Generate the table(s) with algorithm: table_nics_comparison
- specified in the specification file.
-
- :param table: Table to generate.
- :param input_data: Data to process.
- :type table: pandas.Series
- :type input_data: InputData
- """
-
- logging.info(f" Generating the table {table.get(u'title', u'')} ...")
-
- # Transform the data
- logging.info(
- f" Creating the data set for the {table.get(u'type', u'')} "
- f"{table.get(u'title', u'')}."
- )
- data = input_data.filter_data(table, continue_on_error=True)