- ploff.plot(fig, show_link=False, auto_open=False, filename=output_file_name)
-
-
-def table_perf_comparison(table, input_data):
- """Generate the table(s) with algorithm: table_perf_comparison
- specified in the specification file.
-
- :param table: Table to generate.
- :param input_data: Data to process.
- :type table: pandas.Series
- :type input_data: InputData
- """
-
- logging.info(f" Generating the table {table.get(u'title', u'')} ...")
-
- # Transform the data
- logging.info(
- f" Creating the data set for the {table.get(u'type', u'')} "
- f"{table.get(u'title', u'')}."
- )
- data = input_data.filter_data(table, continue_on_error=True)
-
- # Prepare the header of the tables
- try:
- header = [u"Test case", ]
-
- if table[u"include-tests"] == u"MRR":
- hdr_param = u"Rec Rate"
- else:
- hdr_param = u"Thput"
-
- history = table.get(u"history", list())
- for item in history:
- header.extend(
- [
- f"{item[u'title']} {hdr_param} [Mpps]",
- f"{item[u'title']} Stdev [Mpps]"
- ]
- )
- header.extend(
- [
- f"{table[u'reference'][u'title']} {hdr_param} [Mpps]",
- f"{table[u'reference'][u'title']} Stdev [Mpps]",
- f"{table[u'compare'][u'title']} {hdr_param} [Mpps]",
- f"{table[u'compare'][u'title']} Stdev [Mpps]",
- u"Delta [%]",
- u"Stdev of delta [%]"
- ]
- )
- header_str = u",".join(header) + u"\n"
- except (AttributeError, KeyError) as err:
- logging.error(f"The model is invalid, missing parameter: {repr(err)}")
- return
-
- # Prepare data to the table:
- tbl_dict = dict()
- # topo = ""
- for job, builds in table[u"reference"][u"data"].items():
- # topo = u"2n-skx" if u"2n-skx" in job else u""
- for build in builds:
- for tst_name, tst_data in data[job][str(build)].items():
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if (u"across topologies" in table[u"title"].lower() or
- (u" 3n-" in table[u"title"].lower() and
- u" 2n-" in table[u"title"].lower())):
- tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
- if tbl_dict.get(tst_name_mod, None) is None:
- groups = re.search(REGEX_NIC, tst_data[u"parent"])
- nic = groups.group(0) if groups else u""
- name = \
- f"{nic}-{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
- if u"across testbeds" in table[u"title"].lower() or \
- u"across topologies" in table[u"title"].lower():
- name = _tpc_modify_displayed_test_name(name)
- tbl_dict[tst_name_mod] = {
- u"name": name,
- u"ref-data": list(),
- u"cmp-data": list()
- }
- _tpc_insert_data(target=tbl_dict[tst_name_mod][u"ref-data"],
- src=tst_data,
- include_tests=table[u"include-tests"])
-
- replacement = table[u"reference"].get(u"data-replacement", None)
- if replacement:
- create_new_list = True
- rpl_data = input_data.filter_data(
- table, data=replacement, continue_on_error=True)
- for job, builds in replacement.items():
- for build in builds:
- for tst_name, tst_data in rpl_data[job][str(build)].items():
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if (u"across topologies" in table[u"title"].lower() or
- (u" 3n-" in table[u"title"].lower() and
- u" 2n-" in table[u"title"].lower())):
- tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
- if tbl_dict.get(tst_name_mod, None) is None:
- name = \
- f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
- if u"across testbeds" in table[u"title"].lower() or \
- u"across topologies" in table[u"title"].lower():
- name = _tpc_modify_displayed_test_name(name)
- tbl_dict[tst_name_mod] = {
- u"name": name,
- u"ref-data": list(),
- u"cmp-data": list()
- }
- if create_new_list:
- create_new_list = False
- tbl_dict[tst_name_mod][u"ref-data"] = list()
-
- _tpc_insert_data(
- target=tbl_dict[tst_name_mod][u"ref-data"],
- src=tst_data,
- include_tests=table[u"include-tests"]
- )
-
- for job, builds in table[u"compare"][u"data"].items():
- for build in builds:
- for tst_name, tst_data in data[job][str(build)].items():
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if (u"across topologies" in table[u"title"].lower() or
- (u" 3n-" in table[u"title"].lower() and
- u" 2n-" in table[u"title"].lower())):
- tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
- if tbl_dict.get(tst_name_mod, None) is None:
- groups = re.search(REGEX_NIC, tst_data[u"parent"])
- nic = groups.group(0) if groups else u""
- name = \
- f"{nic}-{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
- if u"across testbeds" in table[u"title"].lower() or \
- u"across topologies" in table[u"title"].lower():
- name = _tpc_modify_displayed_test_name(name)
- tbl_dict[tst_name_mod] = {
- u"name": name,
- u"ref-data": list(),
- u"cmp-data": list()
- }
- _tpc_insert_data(
- target=tbl_dict[tst_name_mod][u"cmp-data"],
- src=tst_data,
- include_tests=table[u"include-tests"]
- )
-
- replacement = table[u"compare"].get(u"data-replacement", None)
- if replacement:
- create_new_list = True
- rpl_data = input_data.filter_data(
- table, data=replacement, continue_on_error=True)
- for job, builds in replacement.items():
- for build in builds:
- for tst_name, tst_data in rpl_data[job][str(build)].items():
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if (u"across topologies" in table[u"title"].lower() or
- (u" 3n-" in table[u"title"].lower() and
- u" 2n-" in table[u"title"].lower())):
- tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
- if tbl_dict.get(tst_name_mod, None) is None:
- name = \
- f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
- if u"across testbeds" in table[u"title"].lower() or \
- u"across topologies" in table[u"title"].lower():
- name = _tpc_modify_displayed_test_name(name)
- tbl_dict[tst_name_mod] = {
- u"name": name,
- u"ref-data": list(),
- u"cmp-data": list()
- }
- if create_new_list:
- create_new_list = False
- tbl_dict[tst_name_mod][u"cmp-data"] = list()
-
- _tpc_insert_data(
- target=tbl_dict[tst_name_mod][u"cmp-data"],
- src=tst_data,
- include_tests=table[u"include-tests"]
- )
-
- for item in history:
- for job, builds in item[u"data"].items():
- for build in builds:
- for tst_name, tst_data in data[job][str(build)].items():
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if (u"across topologies" in table[u"title"].lower() or
- (u" 3n-" in table[u"title"].lower() and
- u" 2n-" in table[u"title"].lower())):
- tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
- if tbl_dict.get(tst_name_mod, None) is None:
- continue
- if tbl_dict[tst_name_mod].get(u"history", None) is None:
- tbl_dict[tst_name_mod][u"history"] = OrderedDict()
- if tbl_dict[tst_name_mod][u"history"].\
- get(item[u"title"], None) is None:
- tbl_dict[tst_name_mod][u"history"][item[
- u"title"]] = list()
- try:
- if table[u"include-tests"] == u"MRR":
- res = tst_data[u"result"][u"receive-rate"]
- elif table[u"include-tests"] == u"PDR":
- res = tst_data[u"throughput"][u"PDR"][u"LOWER"]
- elif table[u"include-tests"] == u"NDR":
- res = tst_data[u"throughput"][u"NDR"][u"LOWER"]
- else:
- continue
- tbl_dict[tst_name_mod][u"history"][item[u"title"]].\
- append(res)
- except (TypeError, KeyError):
- pass
-
- tbl_lst = list()
- footnote = False
- for tst_name in tbl_dict:
- item = [tbl_dict[tst_name][u"name"], ]
- if history:
- if tbl_dict[tst_name].get(u"history", None) is not None:
- for hist_data in tbl_dict[tst_name][u"history"].values():
- if hist_data:
- item.append(round(mean(hist_data) / 1000000, 2))
- item.append(round(stdev(hist_data) / 1000000, 2))
- else:
- item.extend([u"Not tested", u"Not tested"])
- else:
- item.extend([u"Not tested", u"Not tested"])
- data_r = tbl_dict[tst_name][u"ref-data"]
- if data_r:
- data_r_mean = mean(data_r)
- item.append(round(data_r_mean / 1000000, 2))
- data_r_stdev = stdev(data_r)
- item.append(round(data_r_stdev / 1000000, 2))
- else:
- data_r_mean = None
- data_r_stdev = None
- item.extend([u"Not tested", u"Not tested"])
- data_c = tbl_dict[tst_name][u"cmp-data"]
- if data_c:
- data_c_mean = mean(data_c)
- item.append(round(data_c_mean / 1000000, 2))
- data_c_stdev = stdev(data_c)
- item.append(round(data_c_stdev / 1000000, 2))
- else:
- data_c_mean = None
- data_c_stdev = None
- item.extend([u"Not tested", u"Not tested"])
- if item[-2] == u"Not tested":
- pass
- elif item[-4] == u"Not tested":
- item.append(u"New in CSIT-2001")
- item.append(u"New in CSIT-2001")
- # elif topo == u"2n-skx" and u"dot1q" in tbl_dict[tst_name][u"name"]:
- # item.append(u"See footnote [1]")
- # footnote = True
- elif data_r_mean and data_c_mean:
- delta, d_stdev = relative_change_stdev(
- data_r_mean, data_c_mean, data_r_stdev, data_c_stdev
- )
- item.append(round(delta, 2))
- item.append(round(d_stdev, 2))
- if (len(item) == len(header)) and (item[-4] != u"Not tested"):
- tbl_lst.append(item)
-
- tbl_lst = _tpc_sort_table(tbl_lst)
-
- # Generate csv tables:
- csv_file = f"{table[u'output-file']}.csv"
- with open(csv_file, u"wt") as file_handler:
- file_handler.write(header_str)
- for test in tbl_lst:
- file_handler.write(u",".join([str(item) for item in test]) + u"\n")
-
- txt_file_name = f"{table[u'output-file']}.txt"
- convert_csv_to_pretty_txt(csv_file, txt_file_name)
-
- if footnote:
- with open(txt_file_name, u'a') as txt_file:
- txt_file.writelines([
- u"\nFootnotes:\n",
- u"[1] CSIT-1908 changed test methodology of dot1q tests in "
- u"2-node testbeds, dot1q encapsulation is now used on both "
- u"links of SUT.\n",
- u" Previously dot1q was used only on a single link with the "
- u"other link carrying untagged Ethernet frames. This changes "
- u"results\n",
- u" in slightly lower throughput in CSIT-1908 for these "
- u"tests. See release notes."
- ])
-
- # Generate html table:
- _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html")
-
-
-def table_perf_comparison_nic(table, input_data):
- """Generate the table(s) with algorithm: table_perf_comparison
- specified in the specification file.
-
- :param table: Table to generate.
- :param input_data: Data to process.
- :type table: pandas.Series
- :type input_data: InputData
- """
-
- logging.info(f" Generating the table {table.get(u'title', u'')} ...")
-
- # Transform the data
- logging.info(
- f" Creating the data set for the {table.get(u'type', u'')} "
- f"{table.get(u'title', u'')}."
- )
- data = input_data.filter_data(table, continue_on_error=True)
-
- # Prepare the header of the tables
- try:
- header = [u"Test case", ]
-
- if table[u"include-tests"] == u"MRR":
- hdr_param = u"Rec Rate"
- else:
- hdr_param = u"Thput"
-
- history = table.get(u"history", list())
- for item in history:
- header.extend(
- [
- f"{item[u'title']} {hdr_param} [Mpps]",
- f"{item[u'title']} Stdev [Mpps]"
- ]
- )
- header.extend(
- [
- f"{table[u'reference'][u'title']} {hdr_param} [Mpps]",
- f"{table[u'reference'][u'title']} Stdev [Mpps]",
- f"{table[u'compare'][u'title']} {hdr_param} [Mpps]",
- f"{table[u'compare'][u'title']} Stdev [Mpps]",
- u"Delta [%]",
- u"Stdev of delta [%]"
- ]
- )
- header_str = u",".join(header) + u"\n"
- except (AttributeError, KeyError) as err:
- logging.error(f"The model is invalid, missing parameter: {repr(err)}")
- return
-
- # Prepare data to the table:
- tbl_dict = dict()
- # topo = u""
- for job, builds in table[u"reference"][u"data"].items():
- # topo = u"2n-skx" if u"2n-skx" in job else u""
- for build in builds:
- for tst_name, tst_data in data[job][str(build)].items():
- if table[u"reference"][u"nic"] not in tst_data[u"tags"]:
- continue
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if (u"across topologies" in table[u"title"].lower() or
- (u" 3n-" in table[u"title"].lower() and
- u" 2n-" in table[u"title"].lower())):
- tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
- if tbl_dict.get(tst_name_mod, None) is None:
- name = f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
- if u"across testbeds" in table[u"title"].lower() or \
- u"across topologies" in table[u"title"].lower():
- name = _tpc_modify_displayed_test_name(name)
- tbl_dict[tst_name_mod] = {
- u"name": name,
- u"ref-data": list(),
- u"cmp-data": list()
- }
- _tpc_insert_data(
- target=tbl_dict[tst_name_mod][u"ref-data"],
- src=tst_data,
- include_tests=table[u"include-tests"]
- )
-
- replacement = table[u"reference"].get(u"data-replacement", None)
- if replacement:
- create_new_list = True
- rpl_data = input_data.filter_data(
- table, data=replacement, continue_on_error=True)
- for job, builds in replacement.items():
- for build in builds:
- for tst_name, tst_data in rpl_data[job][str(build)].items():
- if table[u"reference"][u"nic"] not in tst_data[u"tags"]:
- continue
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if (u"across topologies" in table[u"title"].lower() or
- (u" 3n-" in table[u"title"].lower() and
- u" 2n-" in table[u"title"].lower())):
- tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
- if tbl_dict.get(tst_name_mod, None) is None:
- name = \
- f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
- if u"across testbeds" in table[u"title"].lower() or \
- u"across topologies" in table[u"title"].lower():
- name = _tpc_modify_displayed_test_name(name)
- tbl_dict[tst_name_mod] = {
- u"name": name,
- u"ref-data": list(),
- u"cmp-data": list()
- }
- if create_new_list:
- create_new_list = False
- tbl_dict[tst_name_mod][u"ref-data"] = list()
-
- _tpc_insert_data(
- target=tbl_dict[tst_name_mod][u"ref-data"],
- src=tst_data,
- include_tests=table[u"include-tests"]
- )
-
- for job, builds in table[u"compare"][u"data"].items():
- for build in builds:
- for tst_name, tst_data in data[job][str(build)].items():
- if table[u"compare"][u"nic"] not in tst_data[u"tags"]:
- continue
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if (u"across topologies" in table[u"title"].lower() or
- (u" 3n-" in table[u"title"].lower() and
- u" 2n-" in table[u"title"].lower())):
- tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
- if tbl_dict.get(tst_name_mod, None) is None:
- name = f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
- if u"across testbeds" in table[u"title"].lower() or \
- u"across topologies" in table[u"title"].lower():
- name = _tpc_modify_displayed_test_name(name)
- tbl_dict[tst_name_mod] = {
- u"name": name,
- u"ref-data": list(),
- u"cmp-data": list()
- }
- _tpc_insert_data(
- target=tbl_dict[tst_name_mod][u"cmp-data"],
- src=tst_data,
- include_tests=table[u"include-tests"]
- )
-
- replacement = table[u"compare"].get(u"data-replacement", None)
- if replacement:
- create_new_list = True
- rpl_data = input_data.filter_data(
- table, data=replacement, continue_on_error=True)
- for job, builds in replacement.items():
- for build in builds:
- for tst_name, tst_data in rpl_data[job][str(build)].items():
- if table[u"compare"][u"nic"] not in tst_data[u"tags"]:
- continue
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if (u"across topologies" in table[u"title"].lower() or
- (u" 3n-" in table[u"title"].lower() and
- u" 2n-" in table[u"title"].lower())):
- tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
- if tbl_dict.get(tst_name_mod, None) is None:
- name = \
- f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
- if u"across testbeds" in table[u"title"].lower() or \
- u"across topologies" in table[u"title"].lower():
- name = _tpc_modify_displayed_test_name(name)
- tbl_dict[tst_name_mod] = {
- u"name": name,
- u"ref-data": list(),
- u"cmp-data": list()
- }
- if create_new_list:
- create_new_list = False
- tbl_dict[tst_name_mod][u"cmp-data"] = list()
-
- _tpc_insert_data(
- target=tbl_dict[tst_name_mod][u"cmp-data"],
- src=tst_data,
- include_tests=table[u"include-tests"]
- )
-
- for item in history:
- for job, builds in item[u"data"].items():
- for build in builds:
- for tst_name, tst_data in data[job][str(build)].items():
- if item[u"nic"] not in tst_data[u"tags"]:
- continue
- tst_name_mod = _tpc_modify_test_name(tst_name)
- if (u"across topologies" in table[u"title"].lower() or
- (u" 3n-" in table[u"title"].lower() and
- u" 2n-" in table[u"title"].lower())):
- tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
- if tbl_dict.get(tst_name_mod, None) is None:
- continue
- if tbl_dict[tst_name_mod].get(u"history", None) is None:
- tbl_dict[tst_name_mod][u"history"] = OrderedDict()
- if tbl_dict[tst_name_mod][u"history"].\
- get(item[u"title"], None) is None:
- tbl_dict[tst_name_mod][u"history"][item[
- u"title"]] = list()
- try:
- if table[u"include-tests"] == u"MRR":
- res = tst_data[u"result"][u"receive-rate"]
- elif table[u"include-tests"] == u"PDR":
- res = tst_data[u"throughput"][u"PDR"][u"LOWER"]
- elif table[u"include-tests"] == u"NDR":
- res = tst_data[u"throughput"][u"NDR"][u"LOWER"]
- else:
- continue
- tbl_dict[tst_name_mod][u"history"][item[u"title"]].\
- append(res)
- except (TypeError, KeyError):
- pass
-
- tbl_lst = list()
- footnote = False
- for tst_name in tbl_dict:
- item = [tbl_dict[tst_name][u"name"], ]
- if history:
- if tbl_dict[tst_name].get(u"history", None) is not None:
- for hist_data in tbl_dict[tst_name][u"history"].values():
- if hist_data:
- item.append(round(mean(hist_data) / 1000000, 2))
- item.append(round(stdev(hist_data) / 1000000, 2))
- else:
- item.extend([u"Not tested", u"Not tested"])
- else:
- item.extend([u"Not tested", u"Not tested"])
- data_r = tbl_dict[tst_name][u"ref-data"]
- if data_r:
- data_r_mean = mean(data_r)
- item.append(round(data_r_mean / 1000000, 2))
- data_r_stdev = stdev(data_r)
- item.append(round(data_r_stdev / 1000000, 2))
- else:
- data_r_mean = None
- data_r_stdev = None
- item.extend([u"Not tested", u"Not tested"])
- data_c = tbl_dict[tst_name][u"cmp-data"]
- if data_c:
- data_c_mean = mean(data_c)
- item.append(round(data_c_mean / 1000000, 2))
- data_c_stdev = stdev(data_c)
- item.append(round(data_c_stdev / 1000000, 2))
- else:
- data_c_mean = None
- data_c_stdev = None
- item.extend([u"Not tested", u"Not tested"])
- if item[-2] == u"Not tested":
- pass
- elif item[-4] == u"Not tested":
- item.append(u"New in CSIT-2001")
- item.append(u"New in CSIT-2001")
- # elif topo == u"2n-skx" and u"dot1q" in tbl_dict[tst_name][u"name"]:
- # item.append(u"See footnote [1]")
- # footnote = True
- elif data_r_mean and data_c_mean:
- delta, d_stdev = relative_change_stdev(
- data_r_mean, data_c_mean, data_r_stdev, data_c_stdev
- )
- item.append(round(delta, 2))
- item.append(round(d_stdev, 2))
- if (len(item) == len(header)) and (item[-4] != u"Not tested"):
- tbl_lst.append(item)
-
- tbl_lst = _tpc_sort_table(tbl_lst)
-
- # Generate csv tables:
- csv_file = f"{table[u'output-file']}.csv"
- with open(csv_file, u"wt") as file_handler:
- file_handler.write(header_str)
- for test in tbl_lst:
- file_handler.write(u",".join([str(item) for item in test]) + u"\n")
-
- txt_file_name = f"{table[u'output-file']}.txt"
- convert_csv_to_pretty_txt(csv_file, txt_file_name)
-
- if footnote:
- with open(txt_file_name, u'a') as txt_file:
- txt_file.writelines([
- u"\nFootnotes:\n",
- u"[1] CSIT-1908 changed test methodology of dot1q tests in "
- u"2-node testbeds, dot1q encapsulation is now used on both "
- u"links of SUT.\n",
- u" Previously dot1q was used only on a single link with the "
- u"other link carrying untagged Ethernet frames. This changes "
- u"results\n",
- u" in slightly lower throughput in CSIT-1908 for these "
- u"tests. See release notes."
- ])
-
- # Generate html table:
- _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html")
-
-
-def table_nics_comparison(table, input_data):
- """Generate the table(s) with algorithm: table_nics_comparison
- specified in the specification file.
-
- :param table: Table to generate.
- :param input_data: Data to process.
- :type table: pandas.Series
- :type input_data: InputData
- """
-
- logging.info(f" Generating the table {table.get(u'title', u'')} ...")
-
- # Transform the data
- logging.info(
- f" Creating the data set for the {table.get(u'type', u'')} "
- f"{table.get(u'title', u'')}."
- )
- data = input_data.filter_data(table, continue_on_error=True)
-
- # Prepare the header of the tables
- try:
- header = [u"Test case", ]
-
- if table[u"include-tests"] == u"MRR":
- hdr_param = u"Rec Rate"
- else:
- hdr_param = u"Thput"
-
- header.extend(
- [
- f"{table[u'reference'][u'title']} {hdr_param} [Mpps]",
- f"{table[u'reference'][u'title']} Stdev [Mpps]",
- f"{table[u'compare'][u'title']} {hdr_param} [Mpps]",
- f"{table[u'compare'][u'title']} Stdev [Mpps]",
- u"Delta [%]",
- u"Stdev of delta [%]"
- ]
- )
-
- except (AttributeError, KeyError) as err:
- logging.error(f"The model is invalid, missing parameter: {repr(err)}")
- return