-# Copyright (c) 2019 Cisco and/or its affiliates.
+# Copyright (c) 2020 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
import pandas as pd
from numpy import nan, isnan
+from yaml import load, FullLoader, YAMLError
-from pal_utils import mean, stdev, relative_change, classify_anomalies, \
+from pal_utils import mean, stdev, classify_anomalies, \
convert_csv_to_pretty_txt, relative_change_stdev
-REGEX_NIC = re.compile(r'\d*ge\dp\d\D*\d*')
+REGEX_NIC = re.compile(r'(\d*ge\dp\d\D*\d*[a-z]*)')
def generate_tables(spec, data):
"""
generator = {
- u"table_details": table_details,
u"table_merged_details": table_merged_details,
u"table_perf_comparison": table_perf_comparison,
u"table_perf_comparison_nic": table_perf_comparison_nic,
u"table_perf_trending_dash_html": table_perf_trending_dash_html,
u"table_last_failed_tests": table_last_failed_tests,
u"table_failed_tests": table_failed_tests,
- u"table_failed_tests_html": table_failed_tests_html
+ u"table_failed_tests_html": table_failed_tests_html,
+ u"table_oper_data_html": table_oper_data_html
}
logging.info(u"Generating the tables ...")
logging.info(u"Done.")
-def table_details(table, input_data):
- """Generate the table(s) with algorithm: table_detailed_test_results
+def table_oper_data_html(table, input_data):
+ """Generate the table(s) with algorithm: html_table_oper_data
specified in the specification file.
:param table: Table to generate.
"""
logging.info(f" Generating the table {table.get(u'title', u'')} ...")
-
# Transform the data
logging.info(
f" Creating the data set for the {table.get(u'type', u'')} "
f"{table.get(u'title', u'')}."
)
- data = input_data.filter_data(table)
+ data = input_data.filter_data(
+ table,
+ params=[u"name", u"parent", u"show-run", u"type"],
+ continue_on_error=True
+ )
+ if data.empty:
+ return
+ data = input_data.merge_data(data)
- # Prepare the header of the tables
- header = list()
- for column in table[u"columns"]:
- header.append(
- u'"{0}"'.format(str(column[u"title"]).replace(u'"', u'""'))
+ sort_tests = table.get(u"sort", None)
+ if sort_tests:
+ args = dict(
+ inplace=True,
+ ascending=(sort_tests == u"ascending")
)
+ data.sort_index(**args)
- # Generate the data for the table according to the model in the table
- # specification
- job = list(table[u"data"].keys())[0]
- build = str(table[u"data"][job][0])
- try:
- suites = input_data.suites(job, build)
- except KeyError:
- logging.error(
- u" No data available. The table will not be generated."
- )
+ suites = input_data.filter_data(
+ table,
+ continue_on_error=True,
+ data_set=u"suites"
+ )
+ if suites.empty:
return
+ suites = input_data.merge_data(suites)
- for suite in suites.values:
- # Generate data
- suite_name = suite[u"name"]
- table_lst = list()
- for test in data[job][build].keys():
- if data[job][build][test][u"parent"] not in suite_name:
+ def _generate_html_table(tst_data):
+ """Generate an HTML table with operational data for the given test.
+
+ :param tst_data: Test data to be used to generate the table.
+ :type tst_data: pandas.Series
+ :returns: HTML table with operational data.
+ :rtype: str
+ """
+
+ colors = {
+ u"header": u"#7eade7",
+ u"empty": u"#ffffff",
+ u"body": (u"#e9f1fb", u"#d4e4f7")
+ }
+
+ tbl = ET.Element(u"table", attrib=dict(width=u"100%", border=u"0"))
+
+ trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"header"]))
+ thead = ET.SubElement(
+ trow, u"th", attrib=dict(align=u"left", colspan=u"6")
+ )
+ thead.text = tst_data[u"name"]
+
+ trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]))
+ thead = ET.SubElement(
+ trow, u"th", attrib=dict(align=u"left", colspan=u"6")
+ )
+ thead.text = u"\t"
+
+ if tst_data.get(u"show-run", u"No Data") == u"No Data":
+ trow = ET.SubElement(
+ tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
+ )
+ tcol = ET.SubElement(
+ trow, u"td", attrib=dict(align=u"left", colspan=u"6")
+ )
+ tcol.text = u"No Data"
+
+ trow = ET.SubElement(
+ tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
+ )
+ thead = ET.SubElement(
+ trow, u"th", attrib=dict(align=u"left", colspan=u"6")
+ )
+ font = ET.SubElement(
+ thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff")
+ )
+ font.text = u"."
+ return str(ET.tostring(tbl, encoding=u"unicode"))
+
+ tbl_hdr = (
+ u"Name",
+ u"Nr of Vectors",
+ u"Nr of Packets",
+ u"Suspends",
+ u"Cycles per Packet",
+ u"Average Vector Size"
+ )
+
+ for dut_data in tst_data[u"show-run"].values():
+ trow = ET.SubElement(
+ tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
+ )
+ tcol = ET.SubElement(
+ trow, u"td", attrib=dict(align=u"left", colspan=u"6")
+ )
+ if dut_data.get(u"threads", None) is None:
+ tcol.text = u"No Data"
continue
- row_lst = list()
- for column in table[u"columns"]:
- try:
- col_data = str(data[job][build][test][column[
- u"data"].split(" ")[1]]).replace(u'"', u'""')
- if column[u"data"].split(u" ")[1] in \
- (u"conf-history", u"show-run"):
- col_data = col_data.replace(u" |br| ", u"", )
- col_data = f" |prein| {col_data[:-5]} |preout| "
- row_lst.append(f'"{col_data}"')
- except KeyError:
- row_lst.append(u"No data")
- table_lst.append(row_lst)
- # Write the data to file
- if table_lst:
- file_name = (
- f"{table[u'output-file']}_{suite_name}"
- f"{table[u'output-file-ext']}"
+ bold = ET.SubElement(tcol, u"b")
+ bold.text = (
+ f"Host IP: {dut_data.get(u'host', '')}, "
+ f"Socket: {dut_data.get(u'socket', '')}"
)
- logging.info(f" Writing file: {file_name}")
- with open(file_name, u"w") as file_handler:
- file_handler.write(u",".join(header) + u"\n")
- for item in table_lst:
- file_handler.write(u",".join(item) + u"\n")
+ trow = ET.SubElement(
+ tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
+ )
+ thead = ET.SubElement(
+ trow, u"th", attrib=dict(align=u"left", colspan=u"6")
+ )
+ thead.text = u"\t"
+
+ for thread_nr, thread in dut_data[u"threads"].items():
+ trow = ET.SubElement(
+ tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
+ )
+ tcol = ET.SubElement(
+ trow, u"td", attrib=dict(align=u"left", colspan=u"6")
+ )
+ bold = ET.SubElement(tcol, u"b")
+ bold.text = u"main" if thread_nr == 0 else f"worker_{thread_nr}"
+ trow = ET.SubElement(
+ tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
+ )
+ for idx, col in enumerate(tbl_hdr):
+ tcol = ET.SubElement(
+ trow, u"td",
+ attrib=dict(align=u"right" if idx else u"left")
+ )
+ font = ET.SubElement(
+ tcol, u"font", attrib=dict(size=u"2")
+ )
+ bold = ET.SubElement(font, u"b")
+ bold.text = col
+ for row_nr, row in enumerate(thread):
+ trow = ET.SubElement(
+ tbl, u"tr",
+ attrib=dict(bgcolor=colors[u"body"][row_nr % 2])
+ )
+ for idx, col in enumerate(row):
+ tcol = ET.SubElement(
+ trow, u"td",
+ attrib=dict(align=u"right" if idx else u"left")
+ )
+ font = ET.SubElement(
+ tcol, u"font", attrib=dict(size=u"2")
+ )
+ if isinstance(col, float):
+ font.text = f"{col:.2f}"
+ else:
+ font.text = str(col)
+ trow = ET.SubElement(
+ tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"])
+ )
+ thead = ET.SubElement(
+ trow, u"th", attrib=dict(align=u"left", colspan=u"6")
+ )
+ thead.text = u"\t"
+
+ trow = ET.SubElement(tbl, u"tr", attrib=dict(bgcolor=colors[u"empty"]))
+ thead = ET.SubElement(
+ trow, u"th", attrib=dict(align=u"left", colspan=u"6")
+ )
+ font = ET.SubElement(
+ thead, u"font", attrib=dict(size=u"12px", color=u"#ffffff")
+ )
+ font.text = u"."
+
+ return str(ET.tostring(tbl, encoding=u"unicode"))
+ for suite in suites.values:
+ html_table = str()
+ for test_data in data.values:
+ if test_data[u"parent"] not in suite[u"name"]:
+ continue
+ html_table += _generate_html_table(test_data)
+ if not html_table:
+ continue
+ try:
+ file_name = f"{table[u'output-file']}{suite[u'name']}.rst"
+ with open(f"{file_name}", u'w') as html_file:
+ logging.info(f" Writing file: {file_name}")
+ html_file.write(u".. raw:: html\n\n\t")
+ html_file.write(html_table)
+ html_file.write(u"\n\t<p><br><br></p>\n")
+ except KeyError:
+ logging.warning(u"The output file is not defined.")
+ return
logging.info(u" Done.")
"""
logging.info(f" Generating the table {table.get(u'title', u'')} ...")
+
# Transform the data
logging.info(
f" Creating the data set for the {table.get(u'type', u'')} "
)
data = input_data.filter_data(table, continue_on_error=True)
data = input_data.merge_data(data)
- data.sort_index(inplace=True)
- logging.info(
- f" Creating the data set for the {table.get(u'type', u'')} "
- f"{table.get(u'title', u'')}."
- )
+ sort_tests = table.get(u"sort", None)
+ if sort_tests:
+ args = dict(
+ inplace=True,
+ ascending=(sort_tests == u"ascending")
+ )
+ data.sort_index(**args)
+
suites = input_data.filter_data(
table, continue_on_error=True, data_set=u"suites")
suites = input_data.merge_data(suites)
try:
col_data = str(data[test][column[
u"data"].split(u" ")[1]]).replace(u'"', u'""')
+ # Do not include tests with "Test Failed" in test message
+ if u"Test Failed" in col_data:
+ continue
col_data = col_data.replace(
u"No Data", u"Not Captured "
)
- if column[u"data"].split(u" ")[1] in \
- (u"conf-history", u"show-run"):
+ if column[u"data"].split(u" ")[1] in (u"name", ):
+ if len(col_data) > 30:
+ col_data_lst = col_data.split(u"-")
+ half = int(len(col_data_lst) / 2)
+ col_data = f"{u'-'.join(col_data_lst[:half])}" \
+ f"- |br| " \
+ f"{u'-'.join(col_data_lst[half:])}"
+ col_data = f" |prein| {col_data} |preout| "
+ elif column[u"data"].split(u" ")[1] in (u"msg", ):
+ # Temporary solution: remove NDR results from message:
+ if bool(table.get(u'remove-ndr', False)):
+ try:
+ col_data = col_data.split(u" |br| ", 1)[1]
+ except IndexError:
+ pass
+ col_data = f" |prein| {col_data} |preout| "
+ elif column[u"data"].split(u" ")[1] in \
+ (u"conf-history", u"show-run"):
col_data = col_data.replace(u" |br| ", u"", 1)
col_data = f" |prein| {col_data[:-5]} |preout| "
row_lst.append(f'"{col_data}"')
except KeyError:
row_lst.append(u'"Not captured"')
- table_lst.append(row_lst)
+ if len(row_lst) == len(table[u"columns"]):
+ table_lst.append(row_lst)
# Write the data to file
if table_lst:
- file_name = (
- f"{table[u'output-file']}_{suite_name}"
- f"{table[u'output-file-ext']}"
- )
+ separator = u"" if table[u'output-file'].endswith(u"/") else u"_"
+ file_name = f"{table[u'output-file']}{separator}{suite_name}.csv"
logging.info(f" Writing file: {file_name}")
- with open(file_name, u"w") as file_handler:
+ with open(file_name, u"wt") as file_handler:
file_handler.write(u",".join(header) + u"\n")
for item in table_lst:
file_handler.write(u",".join(item) + u"\n")
"""
try:
if include_tests == u"MRR":
- target.append(src[u"result"][u"receive-rate"])
+ target.append(
+ (
+ src[u"result"][u"receive-rate"],
+ src[u"result"][u"receive-stdev"]
+ )
+ )
elif include_tests == u"PDR":
target.append(src[u"throughput"][u"PDR"][u"LOWER"])
elif include_tests == u"NDR":
:rtype: list
"""
-
tbl_new = list()
tbl_see = list()
tbl_delta = list()
# Sort the tables:
tbl_new.sort(key=lambda rel: rel[0], reverse=False)
tbl_see.sort(key=lambda rel: rel[0], reverse=False)
- tbl_see.sort(key=lambda rel: rel[-1], reverse=False)
- tbl_delta.sort(key=lambda rel: rel[-1], reverse=True)
+ tbl_see.sort(key=lambda rel: rel[-2], reverse=False)
+ tbl_delta.sort(key=lambda rel: rel[0], reverse=False)
+ tbl_delta.sort(key=lambda rel: rel[-2], reverse=True)
# Put the tables together:
table = list()
- table.extend(tbl_new)
+ # We do not want "New in CSIT":
+ # table.extend(tbl_new)
table.extend(tbl_see)
table.extend(tbl_delta)
return table
-def _tpc_generate_html_table(header, data, output_file_name):
+def _tpc_generate_html_table(header, data, out_file_name, legend=u"",
+ footnote=u""):
"""Generate html table from input data with simple sorting possibility.
:param header: Table header.
Inner lists are rows in the table. All inner lists must be of the same
length. The length of these lists must be the same as the length of the
header.
- :param output_file_name: The name (relative or full path) where the
+ :param out_file_name: The name (relative or full path) where the
generated html table is written.
+ :param legend: The legend to display below the table.
+ :param footnote: The footnote to display below the table (and legend).
:type header: list
:type data: list of lists
- :type output_file_name: str
+ :type out_file_name: str
+ :type legend: str
+ :type footnote: str
"""
+ try:
+ idx = header.index(u"Test Case")
+ except ValueError:
+ idx = 0
+ params = {
+ u"align-hdr": ([u"left", u"center"], [u"left", u"left", u"center"]),
+ u"align-itm": ([u"left", u"right"], [u"left", u"left", u"right"]),
+ u"width": ([28, 9], [4, 24, 10])
+ }
+
df_data = pd.DataFrame(data, columns=header)
df_sorted = [df_data.sort_values(
- by=[key, header[0]], ascending=[True, True]
- if key != header[0] else [False, True]) for key in header]
+ by=[key, header[idx]], ascending=[True, True]
+ if key != header[idx] else [False, True]) for key in header]
df_sorted_rev = [df_data.sort_values(
- by=[key, header[0]], ascending=[False, True]
- if key != header[0] else [True, True]) for key in header]
+ by=[key, header[idx]], ascending=[False, True]
+ if key != header[idx] else [True, True]) for key in header]
df_sorted.extend(df_sorted_rev)
fill_color = [[u"#d4e4f7" if idx % 2 else u"#e9f1fb"
table_header = dict(
values=[f"<b>{item}</b>" for item in header],
fill_color=u"#7eade7",
- align=[u"left", u"center"]
+ align=params[u"align-hdr"][idx]
)
fig = go.Figure()
columns = [table.get(col) for col in header]
fig.add_trace(
go.Table(
- columnwidth=[30, 10],
+ columnwidth=params[u"width"][idx],
header=table_header,
cells=dict(
values=columns,
fill_color=fill_color,
- align=[u"left", u"right"]
+ align=params[u"align-itm"][idx]
)
)
)
go.layout.Updatemenu(
type=u"dropdown",
direction=u"down",
- x=0.03,
+ x=0.0,
xanchor=u"left",
y=1.045,
yanchor=u"top",
active=len(menu_items) - 1,
buttons=list(buttons)
)
- ],
- annotations=[
- go.layout.Annotation(
- text=u"<b>Sort by:</b>",
- x=0,
- xref=u"paper",
- y=1.035,
- yref=u"paper",
- align=u"left",
- showarrow=False
- )
]
)
- ploff.plot(fig, show_link=False, auto_open=False, filename=output_file_name)
+ ploff.plot(
+ fig,
+ show_link=False,
+ auto_open=False,
+ filename=f"{out_file_name}_in.html"
+ )
+
+ file_name = out_file_name.split(u"/")[-1]
+ if u"vpp" in out_file_name:
+ path = u"_tmp/src/vpp_performance_tests/comparisons/"
+ else:
+ path = u"_tmp/src/dpdk_performance_tests/comparisons/"
+ with open(f"{path}{file_name}.rst", u"wt") as rst_file:
+ rst_file.write(
+ u"\n"
+ u".. |br| raw:: html\n\n <br />\n\n\n"
+ u".. |prein| raw:: html\n\n <pre>\n\n\n"
+ u".. |preout| raw:: html\n\n </pre>\n\n"
+ )
+ rst_file.write(
+ u".. raw:: html\n\n"
+ f' <iframe frameborder="0" scrolling="no" '
+ f'width="1600" height="1000" '
+ f'src="../..{out_file_name.replace(u"_build", u"")}_in.html">'
+ f'</iframe>\n\n'
+ )
+ if legend:
+ rst_file.write(legend[1:].replace(u"\n", u" |br| "))
+ if footnote:
+ rst_file.write(footnote.replace(u"\n", u" |br| ")[1:])
def table_perf_comparison(table, input_data):
# Prepare the header of the tables
try:
- header = [u"Test case", ]
+ header = [u"Test Case", ]
+ legend = u"\nLegend:\n"
- if table[u"include-tests"] == u"MRR":
- hdr_param = u"Rec Rate"
- else:
- hdr_param = u"Thput"
+ rca_data = None
+ rca = table.get(u"rca", None)
+ if rca:
+ try:
+ with open(rca.get(u"data-file", ""), u"r") as rca_file:
+ rca_data = load(rca_file, Loader=FullLoader)
+ header.insert(0, rca.get(u"title", "RCA"))
+ legend += (
+ u"RCA: Reference to the Root Cause Analysis, see below.\n"
+ )
+ except (YAMLError, IOError) as err:
+ logging.warning(repr(err))
history = table.get(u"history", list())
for item in history:
header.extend(
[
- f"{item[u'title']} {hdr_param} [Mpps]",
- f"{item[u'title']} Stdev [Mpps]"
+ f"{item[u'title']} Avg({table[u'include-tests']})",
+ f"{item[u'title']} Stdev({table[u'include-tests']})"
]
)
+ legend += (
+ f"{item[u'title']} Avg({table[u'include-tests']}): "
+ f"Mean value of {table[u'include-tests']} [Mpps] computed from "
+ f"a series of runs of the listed tests executed against "
+ f"{item[u'title']}.\n"
+ f"{item[u'title']} Stdev({table[u'include-tests']}): "
+ f"Standard deviation value of {table[u'include-tests']} [Mpps] "
+ f"computed from a series of runs of the listed tests executed "
+ f"against {item[u'title']}.\n"
+ )
header.extend(
[
- f"{table[u'reference'][u'title']} {hdr_param} [Mpps]",
- f"{table[u'reference'][u'title']} Stdev [Mpps]",
- f"{table[u'compare'][u'title']} {hdr_param} [Mpps]",
- f"{table[u'compare'][u'title']} Stdev [Mpps]",
- u"Delta [%]"
+ f"{table[u'reference'][u'title']} "
+ f"Avg({table[u'include-tests']})",
+ f"{table[u'reference'][u'title']} "
+ f"Stdev({table[u'include-tests']})",
+ f"{table[u'compare'][u'title']} "
+ f"Avg({table[u'include-tests']})",
+ f"{table[u'compare'][u'title']} "
+ f"Stdev({table[u'include-tests']})",
+ f"Diff({table[u'reference'][u'title']},"
+ f"{table[u'compare'][u'title']})",
+ u"Stdev(Diff)"
]
)
- header_str = u",".join(header) + u"\n"
+ header_str = u";".join(header) + u"\n"
+ legend += (
+ f"{table[u'reference'][u'title']} "
+ f"Avg({table[u'include-tests']}): "
+ f"Mean value of {table[u'include-tests']} [Mpps] computed from a "
+ f"series of runs of the listed tests executed against "
+ f"{table[u'reference'][u'title']}.\n"
+ f"{table[u'reference'][u'title']} "
+ f"Stdev({table[u'include-tests']}): "
+ f"Standard deviation value of {table[u'include-tests']} [Mpps] "
+ f"computed from a series of runs of the listed tests executed "
+ f"against {table[u'reference'][u'title']}.\n"
+ f"{table[u'compare'][u'title']} "
+ f"Avg({table[u'include-tests']}): "
+ f"Mean value of {table[u'include-tests']} [Mpps] computed from a "
+ f"series of runs of the listed tests executed against "
+ f"{table[u'compare'][u'title']}.\n"
+ f"{table[u'compare'][u'title']} "
+ f"Stdev({table[u'include-tests']}): "
+ f"Standard deviation value of {table[u'include-tests']} [Mpps] "
+ f"computed from a series of runs of the listed tests executed "
+ f"against {table[u'compare'][u'title']}.\n"
+ f"Diff({table[u'reference'][u'title']},"
+ f"{table[u'compare'][u'title']}): "
+ f"Percentage change calculated for mean values.\n"
+ u"Stdev(Diff): "
+ u"Standard deviation of percentage change calculated for mean "
+ u"values.\n"
+ u"NT: Not Tested\n"
+ )
except (AttributeError, KeyError) as err:
logging.error(f"The model is invalid, missing parameter: {repr(err)}")
return
# Prepare data to the table:
tbl_dict = dict()
- topo = ""
for job, builds in table[u"reference"][u"data"].items():
- topo = u"2n-skx" if u"2n-skx" in job else u""
for build in builds:
for tst_name, tst_data in data[job][str(build)].items():
tst_name_mod = _tpc_modify_test_name(tst_name)
- if u"across topologies" in table[u"title"].lower():
+ if (u"across topologies" in table[u"title"].lower() or
+ (u" 3n-" in table[u"title"].lower() and
+ u" 2n-" in table[u"title"].lower())):
tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
if tbl_dict.get(tst_name_mod, None) is None:
groups = re.search(REGEX_NIC, tst_data[u"parent"])
src=tst_data,
include_tests=table[u"include-tests"])
+ replacement = table[u"reference"].get(u"data-replacement", None)
+ if replacement:
+ create_new_list = True
+ rpl_data = input_data.filter_data(
+ table, data=replacement, continue_on_error=True)
+ for job, builds in replacement.items():
+ for build in builds:
+ for tst_name, tst_data in rpl_data[job][str(build)].items():
+ tst_name_mod = _tpc_modify_test_name(tst_name)
+ if (u"across topologies" in table[u"title"].lower() or
+ (u" 3n-" in table[u"title"].lower() and
+ u" 2n-" in table[u"title"].lower())):
+ tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
+ if tbl_dict.get(tst_name_mod, None) is None:
+ name = \
+ f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
+ if u"across testbeds" in table[u"title"].lower() or \
+ u"across topologies" in table[u"title"].lower():
+ name = _tpc_modify_displayed_test_name(name)
+ tbl_dict[tst_name_mod] = {
+ u"name": name,
+ u"ref-data": list(),
+ u"cmp-data": list()
+ }
+ if create_new_list:
+ create_new_list = False
+ tbl_dict[tst_name_mod][u"ref-data"] = list()
+
+ _tpc_insert_data(
+ target=tbl_dict[tst_name_mod][u"ref-data"],
+ src=tst_data,
+ include_tests=table[u"include-tests"]
+ )
+
for job, builds in table[u"compare"][u"data"].items():
for build in builds:
for tst_name, tst_data in data[job][str(build)].items():
tst_name_mod = _tpc_modify_test_name(tst_name)
- if u"across topologies" in table[u"title"].lower():
+ if (u"across topologies" in table[u"title"].lower() or
+ (u" 3n-" in table[u"title"].lower() and
+ u" 2n-" in table[u"title"].lower())):
tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
if tbl_dict.get(tst_name_mod, None) is None:
groups = re.search(REGEX_NIC, tst_data[u"parent"])
for build in builds:
for tst_name, tst_data in rpl_data[job][str(build)].items():
tst_name_mod = _tpc_modify_test_name(tst_name)
- if u"across topologies" in table[u"title"].lower():
+ if (u"across topologies" in table[u"title"].lower() or
+ (u" 3n-" in table[u"title"].lower() and
+ u" 2n-" in table[u"title"].lower())):
tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
if tbl_dict.get(tst_name_mod, None) is None:
name = \
for build in builds:
for tst_name, tst_data in data[job][str(build)].items():
tst_name_mod = _tpc_modify_test_name(tst_name)
- if u"across topologies" in table[u"title"].lower():
+ if (u"across topologies" in table[u"title"].lower() or
+ (u" 3n-" in table[u"title"].lower() and
+ u" 2n-" in table[u"title"].lower())):
tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
if tbl_dict.get(tst_name_mod, None) is None:
continue
u"title"]] = list()
try:
if table[u"include-tests"] == u"MRR":
- res = tst_data[u"result"][u"receive-rate"]
+ res = (tst_data[u"result"][u"receive-rate"],
+ tst_data[u"result"][u"receive-stdev"])
elif table[u"include-tests"] == u"PDR":
res = tst_data[u"throughput"][u"PDR"][u"LOWER"]
elif table[u"include-tests"] == u"NDR":
pass
tbl_lst = list()
- footnote = False
for tst_name in tbl_dict:
item = [tbl_dict[tst_name][u"name"], ]
if history:
if tbl_dict[tst_name].get(u"history", None) is not None:
for hist_data in tbl_dict[tst_name][u"history"].values():
if hist_data:
- item.append(round(mean(hist_data) / 1000000, 2))
- item.append(round(stdev(hist_data) / 1000000, 2))
+ if table[u"include-tests"] == u"MRR":
+ item.append(round(hist_data[0][0] / 1e6, 1))
+ item.append(round(hist_data[0][1] / 1e6, 1))
+ else:
+ item.append(round(mean(hist_data) / 1e6, 1))
+ item.append(round(stdev(hist_data) / 1e6, 1))
else:
- item.extend([u"Not tested", u"Not tested"])
+ item.extend([u"NT", u"NT"])
else:
- item.extend([u"Not tested", u"Not tested"])
- data_t = tbl_dict[tst_name][u"ref-data"]
- if data_t:
- item.append(round(mean(data_t) / 1000000, 2))
- item.append(round(stdev(data_t) / 1000000, 2))
+ item.extend([u"NT", u"NT"])
+ data_r = tbl_dict[tst_name][u"ref-data"]
+ if data_r:
+ if table[u"include-tests"] == u"MRR":
+ data_r_mean = data_r[0][0]
+ data_r_stdev = data_r[0][1]
+ else:
+ data_r_mean = mean(data_r)
+ data_r_stdev = stdev(data_r)
+ item.append(round(data_r_mean / 1e6, 1))
+ item.append(round(data_r_stdev / 1e6, 1))
else:
- item.extend([u"Not tested", u"Not tested"])
- data_t = tbl_dict[tst_name][u"cmp-data"]
- if data_t:
- item.append(round(mean(data_t) / 1000000, 2))
- item.append(round(stdev(data_t) / 1000000, 2))
+ data_r_mean = None
+ data_r_stdev = None
+ item.extend([u"NT", u"NT"])
+ data_c = tbl_dict[tst_name][u"cmp-data"]
+ if data_c:
+ if table[u"include-tests"] == u"MRR":
+ data_c_mean = data_c[0][0]
+ data_c_stdev = data_c[0][1]
+ else:
+ data_c_mean = mean(data_c)
+ data_c_stdev = stdev(data_c)
+ item.append(round(data_c_mean / 1e6, 1))
+ item.append(round(data_c_stdev / 1e6, 1))
else:
- item.extend([u"Not tested", u"Not tested"])
- if item[-2] == u"Not tested":
+ data_c_mean = None
+ data_c_stdev = None
+ item.extend([u"NT", u"NT"])
+ if item[-2] == u"NT":
pass
- elif item[-4] == u"Not tested":
- item.append(u"New in CSIT-1908")
- elif topo == u"2n-skx" and u"dot1q" in tbl_dict[tst_name][u"name"]:
- item.append(u"See footnote [1]")
- footnote = True
- elif item[-4] != 0:
- item.append(int(relative_change(float(item[-4]), float(item[-2]))))
- if (len(item) == len(header)) and (item[-3] != u"Not tested"):
+ elif item[-4] == u"NT":
+ item.append(u"New in CSIT-2001")
+ item.append(u"New in CSIT-2001")
+ elif data_r_mean is not None and data_c_mean is not None:
+ delta, d_stdev = relative_change_stdev(
+ data_r_mean, data_c_mean, data_r_stdev, data_c_stdev
+ )
+ try:
+ item.append(round(delta))
+ except ValueError:
+ item.append(delta)
+ try:
+ item.append(round(d_stdev))
+ except ValueError:
+ item.append(d_stdev)
+ if rca_data:
+ rca_nr = rca_data.get(item[0], u"-")
+ item.insert(0, f"[{rca_nr}]" if rca_nr != u"-" else u"-")
+ if (len(item) == len(header)) and (item[-4] != u"NT"):
tbl_lst.append(item)
tbl_lst = _tpc_sort_table(tbl_lst)
# Generate csv tables:
csv_file = f"{table[u'output-file']}.csv"
- with open(csv_file, u"w") as file_handler:
+ with open(csv_file, u"wt") as file_handler:
file_handler.write(header_str)
for test in tbl_lst:
- file_handler.write(u",".join([str(item) for item in test]) + u"\n")
+ file_handler.write(u";".join([str(item) for item in test]) + u"\n")
txt_file_name = f"{table[u'output-file']}.txt"
- convert_csv_to_pretty_txt(csv_file, txt_file_name)
-
- if footnote:
- with open(txt_file_name, u'a') as txt_file:
- txt_file.writelines([
- u"\nFootnotes:\n",
- u"[1] CSIT-1908 changed test methodology of dot1q tests in "
- u"2-node testbeds, dot1q encapsulation is now used on both "
- u"links of SUT.\n",
- u" Previously dot1q was used only on a single link with the "
- u"other link carrying untagged Ethernet frames. This changes "
- u"results\n",
- u" in slightly lower throughput in CSIT-1908 for these "
- u"tests. See release notes."
- ])
+ convert_csv_to_pretty_txt(csv_file, txt_file_name, delimiter=u";")
+
+ footnote = u""
+ with open(txt_file_name, u'a') as txt_file:
+ txt_file.write(legend)
+ if rca_data:
+ footnote = rca_data.get(u"footnote", u"")
+ if footnote:
+ txt_file.write(footnote)
+ txt_file.write(u":END")
# Generate html table:
- _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html")
+ _tpc_generate_html_table(
+ header,
+ tbl_lst,
+ table[u'output-file'],
+ legend=legend,
+ footnote=footnote
+ )
def table_perf_comparison_nic(table, input_data):
# Prepare the header of the tables
try:
- header = [u"Test case", ]
+ header = [u"Test Case", ]
+ legend = u"\nLegend:\n"
- if table[u"include-tests"] == u"MRR":
- hdr_param = u"Rec Rate"
- else:
- hdr_param = u"Thput"
+ rca_data = None
+ rca = table.get(u"rca", None)
+ if rca:
+ try:
+ with open(rca.get(u"data-file", ""), u"r") as rca_file:
+ rca_data = load(rca_file, Loader=FullLoader)
+ header.insert(0, rca.get(u"title", "RCA"))
+ legend += (
+ u"RCA: Reference to the Root Cause Analysis, see below.\n"
+ )
+ except (YAMLError, IOError) as err:
+ logging.warning(repr(err))
history = table.get(u"history", list())
for item in history:
header.extend(
[
- f"{item[u'title']} {hdr_param} [Mpps]",
- f"{item[u'title']} Stdev [Mpps]"
+ f"{item[u'title']} Avg({table[u'include-tests']})",
+ f"{item[u'title']} Stdev({table[u'include-tests']})"
]
)
+ legend += (
+ f"{item[u'title']} Avg({table[u'include-tests']}): "
+ f"Mean value of {table[u'include-tests']} [Mpps] computed from "
+ f"a series of runs of the listed tests executed against "
+ f"{item[u'title']}.\n"
+ f"{item[u'title']} Stdev({table[u'include-tests']}): "
+ f"Standard deviation value of {table[u'include-tests']} [Mpps] "
+ f"computed from a series of runs of the listed tests executed "
+ f"against {item[u'title']}.\n"
+ )
header.extend(
[
- f"{table[u'reference'][u'title']} {hdr_param} [Mpps]",
- f"{table[u'reference'][u'title']} Stdev [Mpps]",
- f"{table[u'compare'][u'title']} {hdr_param} [Mpps]",
- f"{table[u'compare'][u'title']} Stdev [Mpps]",
- u"Delta [%]"
+ f"{table[u'reference'][u'title']} "
+ f"Avg({table[u'include-tests']})",
+ f"{table[u'reference'][u'title']} "
+ f"Stdev({table[u'include-tests']})",
+ f"{table[u'compare'][u'title']} "
+ f"Avg({table[u'include-tests']})",
+ f"{table[u'compare'][u'title']} "
+ f"Stdev({table[u'include-tests']})",
+ f"Diff({table[u'reference'][u'title']},"
+ f"{table[u'compare'][u'title']})",
+ u"Stdev(Diff)"
]
)
- header_str = u",".join(header) + u"\n"
+ header_str = u";".join(header) + u"\n"
+ legend += (
+ f"{table[u'reference'][u'title']} "
+ f"Avg({table[u'include-tests']}): "
+ f"Mean value of {table[u'include-tests']} [Mpps] computed from a "
+ f"series of runs of the listed tests executed against "
+ f"{table[u'reference'][u'title']}.\n"
+ f"{table[u'reference'][u'title']} "
+ f"Stdev({table[u'include-tests']}): "
+ f"Standard deviation value of {table[u'include-tests']} [Mpps] "
+ f"computed from a series of runs of the listed tests executed "
+ f"against {table[u'reference'][u'title']}.\n"
+ f"{table[u'compare'][u'title']} "
+ f"Avg({table[u'include-tests']}): "
+ f"Mean value of {table[u'include-tests']} [Mpps] computed from a "
+ f"series of runs of the listed tests executed against "
+ f"{table[u'compare'][u'title']}.\n"
+ f"{table[u'compare'][u'title']} "
+ f"Stdev({table[u'include-tests']}): "
+ f"Standard deviation value of {table[u'include-tests']} [Mpps] "
+ f"computed from a series of runs of the listed tests executed "
+ f"against {table[u'compare'][u'title']}.\n"
+ f"Diff({table[u'reference'][u'title']},"
+ f"{table[u'compare'][u'title']}): "
+ f"Percentage change calculated for mean values.\n"
+ u"Stdev(Diff): "
+ u"Standard deviation of percentage change calculated for mean "
+ u"values.\n"
+ u"NT: Not Tested\n"
+ )
except (AttributeError, KeyError) as err:
logging.error(f"The model is invalid, missing parameter: {repr(err)}")
return
# Prepare data to the table:
tbl_dict = dict()
- topo = u""
for job, builds in table[u"reference"][u"data"].items():
- topo = u"2n-skx" if u"2n-skx" in job else u""
for build in builds:
for tst_name, tst_data in data[job][str(build)].items():
if table[u"reference"][u"nic"] not in tst_data[u"tags"]:
continue
tst_name_mod = _tpc_modify_test_name(tst_name)
- if u"across topologies" in table[u"title"].lower():
+ if (u"across topologies" in table[u"title"].lower() or
+ (u" 3n-" in table[u"title"].lower() and
+ u" 2n-" in table[u"title"].lower())):
tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
if tbl_dict.get(tst_name_mod, None) is None:
name = f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
include_tests=table[u"include-tests"]
)
+ replacement = table[u"reference"].get(u"data-replacement", None)
+ if replacement:
+ create_new_list = True
+ rpl_data = input_data.filter_data(
+ table, data=replacement, continue_on_error=True)
+ for job, builds in replacement.items():
+ for build in builds:
+ for tst_name, tst_data in rpl_data[job][str(build)].items():
+ if table[u"reference"][u"nic"] not in tst_data[u"tags"]:
+ continue
+ tst_name_mod = _tpc_modify_test_name(tst_name)
+ if (u"across topologies" in table[u"title"].lower() or
+ (u" 3n-" in table[u"title"].lower() and
+ u" 2n-" in table[u"title"].lower())):
+ tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
+ if tbl_dict.get(tst_name_mod, None) is None:
+ name = \
+ f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
+ if u"across testbeds" in table[u"title"].lower() or \
+ u"across topologies" in table[u"title"].lower():
+ name = _tpc_modify_displayed_test_name(name)
+ tbl_dict[tst_name_mod] = {
+ u"name": name,
+ u"ref-data": list(),
+ u"cmp-data": list()
+ }
+ if create_new_list:
+ create_new_list = False
+ tbl_dict[tst_name_mod][u"ref-data"] = list()
+
+ _tpc_insert_data(
+ target=tbl_dict[tst_name_mod][u"ref-data"],
+ src=tst_data,
+ include_tests=table[u"include-tests"]
+ )
+
for job, builds in table[u"compare"][u"data"].items():
for build in builds:
for tst_name, tst_data in data[job][str(build)].items():
if table[u"compare"][u"nic"] not in tst_data[u"tags"]:
continue
tst_name_mod = _tpc_modify_test_name(tst_name)
- if u"across topologies" in table[u"title"].lower():
+ if (u"across topologies" in table[u"title"].lower() or
+ (u" 3n-" in table[u"title"].lower() and
+ u" 2n-" in table[u"title"].lower())):
tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
if tbl_dict.get(tst_name_mod, None) is None:
name = f"{u'-'.join(tst_data[u'name'].split(u'-')[:-1])}"
if table[u"compare"][u"nic"] not in tst_data[u"tags"]:
continue
tst_name_mod = _tpc_modify_test_name(tst_name)
- if u"across topologies" in table[u"title"].lower():
+ if (u"across topologies" in table[u"title"].lower() or
+ (u" 3n-" in table[u"title"].lower() and
+ u" 2n-" in table[u"title"].lower())):
tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
if tbl_dict.get(tst_name_mod, None) is None:
name = \
if item[u"nic"] not in tst_data[u"tags"]:
continue
tst_name_mod = _tpc_modify_test_name(tst_name)
- if u"across topologies" in table[u"title"].lower():
+ if (u"across topologies" in table[u"title"].lower() or
+ (u" 3n-" in table[u"title"].lower() and
+ u" 2n-" in table[u"title"].lower())):
tst_name_mod = tst_name_mod.replace(u"2n1l-", u"")
if tbl_dict.get(tst_name_mod, None) is None:
continue
u"title"]] = list()
try:
if table[u"include-tests"] == u"MRR":
- res = tst_data[u"result"][u"receive-rate"]
+ res = (tst_data[u"result"][u"receive-rate"],
+ tst_data[u"result"][u"receive-stdev"])
elif table[u"include-tests"] == u"PDR":
res = tst_data[u"throughput"][u"PDR"][u"LOWER"]
elif table[u"include-tests"] == u"NDR":
pass
tbl_lst = list()
- footnote = False
for tst_name in tbl_dict:
item = [tbl_dict[tst_name][u"name"], ]
if history:
if tbl_dict[tst_name].get(u"history", None) is not None:
for hist_data in tbl_dict[tst_name][u"history"].values():
if hist_data:
- item.append(round(mean(hist_data) / 1000000, 2))
- item.append(round(stdev(hist_data) / 1000000, 2))
+ if table[u"include-tests"] == u"MRR":
+ item.append(round(hist_data[0][0] / 1e6, 1))
+ item.append(round(hist_data[0][1] / 1e6, 1))
+ else:
+ item.append(round(mean(hist_data) / 1e6, 1))
+ item.append(round(stdev(hist_data) / 1e6, 1))
else:
- item.extend([u"Not tested", u"Not tested"])
+ item.extend([u"NT", u"NT"])
+ else:
+ item.extend([u"NT", u"NT"])
+ data_r = tbl_dict[tst_name][u"ref-data"]
+ if data_r:
+ if table[u"include-tests"] == u"MRR":
+ data_r_mean = data_r[0][0]
+ data_r_stdev = data_r[0][1]
else:
- item.extend([u"Not tested", u"Not tested"])
- data_t = tbl_dict[tst_name][u"ref-data"]
- if data_t:
- item.append(round(mean(data_t) / 1000000, 2))
- item.append(round(stdev(data_t) / 1000000, 2))
+ data_r_mean = mean(data_r)
+ data_r_stdev = stdev(data_r)
+ item.append(round(data_r_mean / 1e6, 1))
+ item.append(round(data_r_stdev / 1e6, 1))
else:
- item.extend([u"Not tested", u"Not tested"])
- data_t = tbl_dict[tst_name][u"cmp-data"]
- if data_t:
- item.append(round(mean(data_t) / 1000000, 2))
- item.append(round(stdev(data_t) / 1000000, 2))
+ data_r_mean = None
+ data_r_stdev = None
+ item.extend([u"NT", u"NT"])
+ data_c = tbl_dict[tst_name][u"cmp-data"]
+ if data_c:
+ if table[u"include-tests"] == u"MRR":
+ data_c_mean = data_c[0][0]
+ data_c_stdev = data_c[0][1]
+ else:
+ data_c_mean = mean(data_c)
+ data_c_stdev = stdev(data_c)
+ item.append(round(data_c_mean / 1e6, 1))
+ item.append(round(data_c_stdev / 1e6, 1))
else:
- item.extend([u"Not tested", u"Not tested"])
- if item[-2] == u"Not tested":
+ data_c_mean = None
+ data_c_stdev = None
+ item.extend([u"NT", u"NT"])
+ if item[-2] == u"NT":
pass
- elif item[-4] == u"Not tested":
- item.append(u"New in CSIT-1908")
- elif topo == u"2n-skx" and u"dot1q" in tbl_dict[tst_name][u"name"]:
- item.append(u"See footnote [1]")
- footnote = True
- elif item[-4] != 0:
- item.append(int(relative_change(float(item[-4]), float(item[-2]))))
- if (len(item) == len(header)) and (item[-3] != u"Not tested"):
+ elif item[-4] == u"NT":
+ item.append(u"New in CSIT-2001")
+ item.append(u"New in CSIT-2001")
+ elif data_r_mean is not None and data_c_mean is not None:
+ delta, d_stdev = relative_change_stdev(
+ data_r_mean, data_c_mean, data_r_stdev, data_c_stdev
+ )
+ try:
+ item.append(round(delta))
+ except ValueError:
+ item.append(delta)
+ try:
+ item.append(round(d_stdev))
+ except ValueError:
+ item.append(d_stdev)
+ if rca_data:
+ rca_nr = rca_data.get(item[0], u"-")
+ item.insert(0, f"[{rca_nr}]" if rca_nr != u"-" else u"-")
+ if (len(item) == len(header)) and (item[-4] != u"NT"):
tbl_lst.append(item)
tbl_lst = _tpc_sort_table(tbl_lst)
# Generate csv tables:
csv_file = f"{table[u'output-file']}.csv"
- with open(csv_file, u"w") as file_handler:
+ with open(csv_file, u"wt") as file_handler:
file_handler.write(header_str)
for test in tbl_lst:
- file_handler.write(u",".join([str(item) for item in test]) + u"\n")
+ file_handler.write(u";".join([str(item) for item in test]) + u"\n")
txt_file_name = f"{table[u'output-file']}.txt"
- convert_csv_to_pretty_txt(csv_file, txt_file_name)
-
- if footnote:
- with open(txt_file_name, u'a') as txt_file:
- txt_file.writelines([
- u"\nFootnotes:\n",
- u"[1] CSIT-1908 changed test methodology of dot1q tests in "
- u"2-node testbeds, dot1q encapsulation is now used on both "
- u"links of SUT.\n",
- u" Previously dot1q was used only on a single link with the "
- u"other link carrying untagged Ethernet frames. This changes "
- u"results\n",
- u" in slightly lower throughput in CSIT-1908 for these "
- u"tests. See release notes."
- ])
+ convert_csv_to_pretty_txt(csv_file, txt_file_name, delimiter=u";")
+
+ footnote = u""
+ with open(txt_file_name, u'a') as txt_file:
+ txt_file.write(legend)
+ if rca_data:
+ footnote = rca_data.get(u"footnote", u"")
+ if footnote:
+ txt_file.write(footnote)
+ txt_file.write(u":END")
# Generate html table:
- _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html")
+ _tpc_generate_html_table(
+ header,
+ tbl_lst,
+ table[u'output-file'],
+ legend=legend,
+ footnote=footnote
+ )
def table_nics_comparison(table, input_data):
# Prepare the header of the tables
try:
- header = [u"Test case", ]
-
- if table[u"include-tests"] == u"MRR":
- hdr_param = u"Rec Rate"
- else:
- hdr_param = u"Thput"
-
- header.extend(
- [
- f"{table[u'reference'][u'title']} {hdr_param} [Mpps]",
- f"{table[u'reference'][u'title']} Stdev [Mpps]",
- f"{table[u'compare'][u'title']} {hdr_param} [Mpps]",
- f"{table[u'compare'][u'title']} Stdev [Mpps]",
- u"Delta [%]"
- ]
+ header = [
+ u"Test Case",
+ f"{table[u'reference'][u'title']} "
+ f"Avg({table[u'include-tests']})",
+ f"{table[u'reference'][u'title']} "
+ f"Stdev({table[u'include-tests']})",
+ f"{table[u'compare'][u'title']} "
+ f"Avg({table[u'include-tests']})",
+ f"{table[u'compare'][u'title']} "
+ f"Stdev({table[u'include-tests']})",
+ f"Diff({table[u'reference'][u'title']},"
+ f"{table[u'compare'][u'title']})",
+ u"Stdev(Diff)"
+ ]
+ legend = (
+ u"\nLegend:\n"
+ f"{table[u'reference'][u'title']} "
+ f"Avg({table[u'include-tests']}): "
+ f"Mean value of {table[u'include-tests']} [Mpps] computed from a "
+ f"series of runs of the listed tests executed using "
+ f"{table[u'reference'][u'title']} NIC.\n"
+ f"{table[u'reference'][u'title']} "
+ f"Stdev({table[u'include-tests']}): "
+ f"Standard deviation value of {table[u'include-tests']} [Mpps] "
+ f"computed from a series of runs of the listed tests executed "
+ f"using {table[u'reference'][u'title']} NIC.\n"
+ f"{table[u'compare'][u'title']} "
+ f"Avg({table[u'include-tests']}): "
+ f"Mean value of {table[u'include-tests']} [Mpps] computed from a "
+ f"series of runs of the listed tests executed using "
+ f"{table[u'compare'][u'title']} NIC.\n"
+ f"{table[u'compare'][u'title']} "
+ f"Stdev({table[u'include-tests']}): "
+ f"Standard deviation value of {table[u'include-tests']} [Mpps] "
+ f"computed from a series of runs of the listed tests executed "
+ f"using {table[u'compare'][u'title']} NIC.\n"
+ f"Diff({table[u'reference'][u'title']},"
+ f"{table[u'compare'][u'title']}): "
+ f"Percentage change calculated for mean values.\n"
+ u"Stdev(Diff): "
+ u"Standard deviation of percentage change calculated for mean "
+ u"values.\n"
+ u":END"
)
except (AttributeError, KeyError) as err:
u"cmp-data": list()
}
try:
- result = None
if table[u"include-tests"] == u"MRR":
- result = tst_data[u"result"][u"receive-rate"]
+ result = (tst_data[u"result"][u"receive-rate"],
+ tst_data[u"result"][u"receive-stdev"])
elif table[u"include-tests"] == u"PDR":
result = tst_data[u"throughput"][u"PDR"][u"LOWER"]
elif table[u"include-tests"] == u"NDR":
tbl_lst = list()
for tst_name in tbl_dict:
item = [tbl_dict[tst_name][u"name"], ]
- data_t = tbl_dict[tst_name][u"ref-data"]
- if data_t:
- item.append(round(mean(data_t) / 1000000, 2))
- item.append(round(stdev(data_t) / 1000000, 2))
+ data_r = tbl_dict[tst_name][u"ref-data"]
+ if data_r:
+ if table[u"include-tests"] == u"MRR":
+ data_r_mean = data_r[0][0]
+ data_r_stdev = data_r[0][1]
+ else:
+ data_r_mean = mean(data_r)
+ data_r_stdev = stdev(data_r)
+ item.append(round(data_r_mean / 1e6, 1))
+ item.append(round(data_r_stdev / 1e6, 1))
else:
+ data_r_mean = None
+ data_r_stdev = None
item.extend([None, None])
- data_t = tbl_dict[tst_name][u"cmp-data"]
- if data_t:
- item.append(round(mean(data_t) / 1000000, 2))
- item.append(round(stdev(data_t) / 1000000, 2))
+ data_c = tbl_dict[tst_name][u"cmp-data"]
+ if data_c:
+ if table[u"include-tests"] == u"MRR":
+ data_c_mean = data_c[0][0]
+ data_c_stdev = data_c[0][1]
+ else:
+ data_c_mean = mean(data_c)
+ data_c_stdev = stdev(data_c)
+ item.append(round(data_c_mean / 1e6, 1))
+ item.append(round(data_c_stdev / 1e6, 1))
else:
+ data_c_mean = None
+ data_c_stdev = None
item.extend([None, None])
- if item[-4] is not None and item[-2] is not None and item[-4] != 0:
- item.append(int(relative_change(float(item[-4]), float(item[-2]))))
- if len(item) == len(header):
+ if data_r_mean is not None and data_c_mean is not None:
+ delta, d_stdev = relative_change_stdev(
+ data_r_mean, data_c_mean, data_r_stdev, data_c_stdev
+ )
+ try:
+ item.append(round(delta))
+ except ValueError:
+ item.append(delta)
+ try:
+ item.append(round(d_stdev))
+ except ValueError:
+ item.append(d_stdev)
tbl_lst.append(item)
# Sort the table according to the relative change
tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
# Generate csv tables:
- with open(f"{table[u'output-file']}.csv", u"w") as file_handler:
- file_handler.write(u",".join(header) + u"\n")
+ with open(f"{table[u'output-file']}.csv", u"wt") as file_handler:
+ file_handler.write(u";".join(header) + u"\n")
for test in tbl_lst:
- file_handler.write(u",".join([str(item) for item in test]) + u"\n")
+ file_handler.write(u";".join([str(item) for item in test]) + u"\n")
convert_csv_to_pretty_txt(f"{table[u'output-file']}.csv",
- f"{table[u'output-file']}.txt")
+ f"{table[u'output-file']}.txt",
+ delimiter=u";")
+
+ with open(f"{table[u'output-file']}.txt", u'a') as txt_file:
+ txt_file.write(legend)
# Generate html table:
- _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html")
+ _tpc_generate_html_table(
+ header,
+ tbl_lst,
+ table[u'output-file'],
+ legend=legend
+ )
def table_soak_vs_ndr(table, input_data):
# Prepare the header of the table
try:
header = [
- u"Test case",
- f"{table[u'reference'][u'title']} Thput [Mpps]",
- f"{table[u'reference'][u'title']} Stdev [Mpps]",
- f"{table[u'compare'][u'title']} Thput [Mpps]",
- f"{table[u'compare'][u'title']} Stdev [Mpps]",
- u"Delta [%]", u"Stdev of delta [%]"
+ u"Test Case",
+ f"Avg({table[u'reference'][u'title']})",
+ f"Stdev({table[u'reference'][u'title']})",
+ f"Avg({table[u'compare'][u'title']})",
+ f"Stdev{table[u'compare'][u'title']})",
+ u"Diff",
+ u"Stdev(Diff)"
]
- header_str = u",".join(header) + u"\n"
+ header_str = u";".join(header) + u"\n"
+ legend = (
+ u"\nLegend:\n"
+ f"Avg({table[u'reference'][u'title']}): "
+ f"Mean value of {table[u'reference'][u'title']} [Mpps] computed "
+ f"from a series of runs of the listed tests.\n"
+ f"Stdev({table[u'reference'][u'title']}): "
+ f"Standard deviation value of {table[u'reference'][u'title']} "
+ f"[Mpps] computed from a series of runs of the listed tests.\n"
+ f"Avg({table[u'compare'][u'title']}): "
+ f"Mean value of {table[u'compare'][u'title']} [Mpps] computed from "
+ f"a series of runs of the listed tests.\n"
+ f"Stdev({table[u'compare'][u'title']}): "
+ f"Standard deviation value of {table[u'compare'][u'title']} [Mpps] "
+ f"computed from a series of runs of the listed tests.\n"
+ f"Diff({table[u'reference'][u'title']},"
+ f"{table[u'compare'][u'title']}): "
+ f"Percentage change calculated for mean values.\n"
+ u"Stdev(Diff): "
+ u"Standard deviation of percentage change calculated for mean "
+ u"values.\n"
+ u":END"
+ )
except (AttributeError, KeyError) as err:
logging.error(f"The model is invalid, missing parameter: {repr(err)}")
return
if tst_data[u"type"] not in (u"NDRPDR", u"MRR", u"BMRR"):
continue
if table[u"include-tests"] == u"MRR":
- result = tst_data[u"result"][u"receive-rate"]
+ result = (tst_data[u"result"][u"receive-rate"],
+ tst_data[u"result"][u"receive-stdev"])
elif table[u"include-tests"] == u"PDR":
result = \
tst_data[u"throughput"][u"PDR"][u"LOWER"]
item = [tbl_dict[tst_name][u"name"], ]
data_r = tbl_dict[tst_name][u"ref-data"]
if data_r:
- data_r_mean = mean(data_r)
- item.append(round(data_r_mean / 1000000, 2))
- data_r_stdev = stdev(data_r)
- item.append(round(data_r_stdev / 1000000, 2))
+ if table[u"include-tests"] == u"MRR":
+ data_r_mean = data_r[0][0]
+ data_r_stdev = data_r[0][1]
+ else:
+ data_r_mean = mean(data_r)
+ data_r_stdev = stdev(data_r)
+ item.append(round(data_r_mean / 1e6, 1))
+ item.append(round(data_r_stdev / 1e6, 1))
else:
data_r_mean = None
data_r_stdev = None
item.extend([None, None])
data_c = tbl_dict[tst_name][u"cmp-data"]
if data_c:
- data_c_mean = mean(data_c)
- item.append(round(data_c_mean / 1000000, 2))
- data_c_stdev = stdev(data_c)
- item.append(round(data_c_stdev / 1000000, 2))
+ if table[u"include-tests"] == u"MRR":
+ data_c_mean = data_c[0][0]
+ data_c_stdev = data_c[0][1]
+ else:
+ data_c_mean = mean(data_c)
+ data_c_stdev = stdev(data_c)
+ item.append(round(data_c_mean / 1e6, 1))
+ item.append(round(data_c_stdev / 1e6, 1))
else:
data_c_mean = None
data_c_stdev = None
item.extend([None, None])
- if data_r_mean and data_c_mean:
+ if data_r_mean is not None and data_c_mean is not None:
delta, d_stdev = relative_change_stdev(
data_r_mean, data_c_mean, data_r_stdev, data_c_stdev)
- item.append(round(delta, 2))
- item.append(round(d_stdev, 2))
+ try:
+ item.append(round(delta))
+ except ValueError:
+ item.append(delta)
+ try:
+ item.append(round(d_stdev))
+ except ValueError:
+ item.append(d_stdev)
tbl_lst.append(item)
# Sort the table according to the relative change
# Generate csv tables:
csv_file = f"{table[u'output-file']}.csv"
- with open(csv_file, u"w") as file_handler:
+ with open(csv_file, u"wt") as file_handler:
file_handler.write(header_str)
for test in tbl_lst:
- file_handler.write(u",".join([str(item) for item in test]) + u"\n")
+ file_handler.write(u";".join([str(item) for item in test]) + u"\n")
- convert_csv_to_pretty_txt(csv_file, f"{table[u'output-file']}.txt")
+ convert_csv_to_pretty_txt(
+ csv_file, f"{table[u'output-file']}.txt", delimiter=u";"
+ )
+ with open(f"{table[u'output-file']}.txt", u'a') as txt_file:
+ txt_file.write(legend)
# Generate html table:
- _tpc_generate_html_table(header, tbl_lst, f"{table[u'output-file']}.html")
+ _tpc_generate_html_table(
+ header,
+ tbl_lst,
+ table[u'output-file'],
+ legend=legend
+ )
def table_perf_trending_dash(table, input_data):
continue
tbl_lst.append(
[tbl_dict[tst_name][u"name"],
- round(last_avg / 1000000, 2),
+ round(last_avg / 1e6, 2),
rel_change_last,
rel_change_long,
classification_lst[-win_size:].count(u"regression"),
file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
logging.info(f" Writing file: {file_name}")
- with open(file_name, u"w") as file_handler:
+ with open(file_name, u"wt") as file_handler:
file_handler.write(header_str)
for test in tbl_sorted:
file_handler.write(u",".join([str(item) for item in test]) + u'\n')
convert_csv_to_pretty_txt(file_name, f"{table[u'output-file']}.txt")
-def _generate_url(base, testbed, test_name):
+def _generate_url(testbed, test_name):
"""Generate URL to a trending plot from the name of the test case.
- :param base: The base part of URL common to all test cases.
:param testbed: The testbed used for testing.
:param test_name: The name of the test case.
- :type base: str
:type testbed: str
:type test_name: str
:returns: The URL to the plot with the trending data for the given test
:rtype str
"""
- url = base
- file_name = u""
- anchor = u".html#"
- feature = u""
-
- if u"lbdpdk" in test_name or u"lbvpp" in test_name:
- file_name = u"link_bonding"
-
- elif u"114b" in test_name and u"vhost" in test_name:
- file_name = u"vts"
-
- elif u"testpmd" in test_name or u"l3fwd" in test_name:
- file_name = u"dpdk"
-
- elif u"memif" in test_name:
- file_name = u"container_memif"
- feature = u"-base"
-
- elif u"srv6" in test_name:
- file_name = u"srv6"
-
- elif u"vhost" in test_name:
- if u"l2xcbase" in test_name or u"l2bdbasemaclrn" in test_name:
- file_name = u"vm_vhost_l2"
- if u"114b" in test_name:
- feature = u""
- elif u"l2xcbase" in test_name and u"x520" in test_name:
- feature = u"-base-l2xc"
- elif u"l2bdbasemaclrn" in test_name and u"x520" in test_name:
- feature = u"-base-l2bd"
- else:
- feature = u"-base"
- elif u"ip4base" in test_name:
- file_name = u"vm_vhost_ip4"
- feature = u"-base"
-
- elif u"ipsecbasetnlsw" in test_name:
- file_name = u"ipsecsw"
- feature = u"-base-scale"
-
- elif u"ipsec" in test_name:
- file_name = u"ipsec"
- feature = u"-base-scale"
- if u"hw-" in test_name:
- file_name = u"ipsechw"
- elif u"sw-" in test_name:
- file_name = u"ipsecsw"
- if u"-int-" in test_name:
- feature = u"-base-scale-int"
- elif u"tnl" in test_name:
- feature = u"-base-scale-tnl"
-
- elif u"ethip4lispip" in test_name or u"ethip4vxlan" in test_name:
- file_name = u"ip4_tunnels"
- feature = u"-base"
-
- elif u"ip4base" in test_name or u"ip4scale" in test_name:
- file_name = u"ip4"
- if u"xl710" in test_name:
- feature = u"-base-scale-features"
- elif u"iacl" in test_name:
- feature = u"-features-iacl"
- elif u"oacl" in test_name:
- feature = u"-features-oacl"
- elif u"snat" in test_name or u"cop" in test_name:
- feature = u"-features"
- else:
- feature = u"-base-scale"
-
- elif u"ip6base" in test_name or u"ip6scale" in test_name:
- file_name = u"ip6"
- feature = u"-base-scale"
-
- elif u"l2xcbase" in test_name or u"l2xcscale" in test_name \
- or u"l2bdbasemaclrn" in test_name or u"l2bdscale" in test_name:
- file_name = u"l2"
- if u"macip" in test_name:
- feature = u"-features-macip"
- elif u"iacl" in test_name:
- feature = u"-features-iacl"
- elif u"oacl" in test_name:
- feature = u"-features-oacl"
- else:
- feature = u"-base-scale"
-
if u"x520" in test_name:
- nic = u"x520-"
+ nic = u"x520"
elif u"x710" in test_name:
- nic = u"x710-"
+ nic = u"x710"
elif u"xl710" in test_name:
- nic = u"xl710-"
+ nic = u"xl710"
elif u"xxv710" in test_name:
- nic = u"xxv710-"
+ nic = u"xxv710"
elif u"vic1227" in test_name:
- nic = u"vic1227-"
+ nic = u"vic1227"
elif u"vic1385" in test_name:
- nic = u"vic1385-"
+ nic = u"vic1385"
elif u"x553" in test_name:
- nic = u"x553-"
+ nic = u"x553"
+ elif u"cx556" in test_name or u"cx556a" in test_name:
+ nic = u"cx556a"
else:
nic = u""
- anchor += nic
if u"64b" in test_name:
- framesize = u"64b"
+ frame_size = u"64b"
elif u"78b" in test_name:
- framesize = u"78b"
+ frame_size = u"78b"
elif u"imix" in test_name:
- framesize = u"imix"
+ frame_size = u"imix"
elif u"9000b" in test_name:
- framesize = u"9000b"
+ frame_size = u"9000b"
elif u"1518b" in test_name:
- framesize = u"1518b"
+ frame_size = u"1518b"
elif u"114b" in test_name:
- framesize = u"114b"
+ frame_size = u"114b"
+ else:
+ frame_size = u""
+
+ if u"1t1c" in test_name or \
+ (u"-1c-" in test_name and
+ testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
+ cores = u"1t1c"
+ elif u"2t2c" in test_name or \
+ (u"-2c-" in test_name and
+ testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
+ cores = u"2t2c"
+ elif u"4t4c" in test_name or \
+ (u"-4c-" in test_name and
+ testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")):
+ cores = u"4t4c"
+ elif u"2t1c" in test_name or \
+ (u"-1c-" in test_name and
+ testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
+ cores = u"2t1c"
+ elif u"4t2c" in test_name or \
+ (u"-2c-" in test_name and
+ testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
+ cores = u"4t2c"
+ elif u"8t4c" in test_name or \
+ (u"-4c-" in test_name and
+ testbed in (u"2n-skx", u"3n-skx", u"2n-clx")):
+ cores = u"8t4c"
else:
- framesize = u""
- anchor += framesize + u"-"
-
- if u"1t1c" in test_name:
- anchor += u"1t1c"
- elif u"2t2c" in test_name:
- anchor += u"2t2c"
- elif u"4t4c" in test_name:
- anchor += u"4t4c"
- elif u"2t1c" in test_name:
- anchor += u"2t1c"
- elif u"4t2c" in test_name:
- anchor += u"4t2c"
- elif u"8t4c" in test_name:
- anchor += u"8t4c"
-
- return url + file_name + u"-" + testbed + u"-" + nic + framesize + \
- feature.replace("-int", u"").replace("-tnl", u"") + anchor + feature
+ cores = u""
+
+ if u"testpmd" in test_name:
+ driver = u"testpmd"
+ elif u"l3fwd" in test_name:
+ driver = u"l3fwd"
+ elif u"avf" in test_name:
+ driver = u"avf"
+ elif u"rdma" in test_name:
+ driver = u"rdma"
+ elif u"dnv" in testbed or u"tsh" in testbed:
+ driver = u"ixgbe"
+ else:
+ driver = u"dpdk"
+
+ if u"acl" in test_name or \
+ u"macip" in test_name or \
+ u"nat" in test_name or \
+ u"policer" in test_name or \
+ u"cop" in test_name:
+ bsf = u"features"
+ elif u"scale" in test_name:
+ bsf = u"scale"
+ elif u"base" in test_name:
+ bsf = u"base"
+ else:
+ bsf = u"base"
+
+ if u"114b" in test_name and u"vhost" in test_name:
+ domain = u"vts"
+ elif u"testpmd" in test_name or u"l3fwd" in test_name:
+ domain = u"dpdk"
+ elif u"memif" in test_name:
+ domain = u"container_memif"
+ elif u"srv6" in test_name:
+ domain = u"srv6"
+ elif u"vhost" in test_name:
+ domain = u"vhost"
+ if u"vppl2xc" in test_name:
+ driver += u"-vpp"
+ else:
+ driver += u"-testpmd"
+ if u"lbvpplacp" in test_name:
+ bsf += u"-link-bonding"
+ elif u"ch" in test_name and u"vh" in test_name and u"vm" in test_name:
+ domain = u"nf_service_density_vnfc"
+ elif u"ch" in test_name and u"mif" in test_name and u"dcr" in test_name:
+ domain = u"nf_service_density_cnfc"
+ elif u"pl" in test_name and u"mif" in test_name and u"dcr" in test_name:
+ domain = u"nf_service_density_cnfp"
+ elif u"ipsec" in test_name:
+ domain = u"ipsec"
+ if u"sw" in test_name:
+ bsf += u"-sw"
+ elif u"hw" in test_name:
+ bsf += u"-hw"
+ elif u"ethip4vxlan" in test_name:
+ domain = u"ip4_tunnels"
+ elif u"ip4base" in test_name or u"ip4scale" in test_name:
+ domain = u"ip4"
+ elif u"ip6base" in test_name or u"ip6scale" in test_name:
+ domain = u"ip6"
+ elif u"l2xcbase" in test_name or \
+ u"l2xcscale" in test_name or \
+ u"l2bdbasemaclrn" in test_name or \
+ u"l2bdscale" in test_name or \
+ u"l2patch" in test_name:
+ domain = u"l2"
+ else:
+ domain = u""
+
+ file_name = u"-".join((domain, testbed, nic)) + u".html#"
+ anchor_name = u"-".join((frame_size, cores, bsf, driver))
+
+ return file_name + anchor_name
def table_perf_trending_dash_html(table, input_data):
tdata,
u"a",
attrib=dict(
- href=_generate_url(
- u"../trending/",
- table.get(u"testbed", None),
- item
- )
+ href=f"../trending/"
+ f"{_generate_url(table.get(u'testbed', ''), item)}"
)
)
ref.text = item
file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
logging.info(f" Writing file: {file_name}")
- with open(file_name, u"w") as file_handler:
+ with open(file_name, u"wt") as file_handler:
for test in tbl_list:
file_handler.write(test + u'\n')
file_name = f"{table[u'output-file']}{table[u'output-file-ext']}"
logging.info(f" Writing file: {file_name}")
- with open(file_name, u"w") as file_handler:
+ with open(file_name, u"wt") as file_handler:
file_handler.write(u",".join(header) + u"\n")
for test in tbl_sorted:
file_handler.write(u",".join([str(item) for item in test]) + u'\n')
tdata,
u"a",
attrib=dict(
- href=_generate_url(
- u"../trending/",
- table.get(u"testbed", None),
- item
- )
+ href=f"../trending/"
+ f"{_generate_url(table.get(u'testbed', ''), item)}"
)
)
ref.text = item
Code Review / csit.git / blobdiff