X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=resources%2Ftools%2Fpresentation%2Fgenerator_tables.py;h=0b063b10670aca8fd8356b7d5d7fea011d124e9c;hb=d24bcf6e10fdee8e6ba1a0ef2f75e69d9e9d0a40;hp=468b9c642db4e790df932f669bd88f744e9896c8;hpb=68ad5eab0fe4aacde4a813cd052a34aab25adc07;p=csit.git
diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py
index 468b9c642d..0b063b1067 100644
--- a/resources/tools/presentation/generator_tables.py
+++ b/resources/tools/presentation/generator_tables.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2020 Cisco and/or its affiliates.
+# Copyright (c) 2021 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
@@ -17,6 +17,7 @@
import logging
import csv
+import math
import re
from collections import OrderedDict
@@ -28,12 +29,13 @@ from copy import deepcopy
import plotly.graph_objects as go
import plotly.offline as ploff
import pandas as pd
+import prettytable
from numpy import nan, isnan
from yaml import load, FullLoader, YAMLError
from pal_utils import mean, stdev, classify_anomalies, \
- convert_csv_to_pretty_txt, relative_change_stdev
+ convert_csv_to_pretty_txt, relative_change_stdev, relative_change
REGEX_NIC = re.compile(r'(\d*ge\dp\d\D*\d*[a-z]*)')
@@ -50,9 +52,6 @@ def generate_tables(spec, data):
generator = {
u"table_merged_details": table_merged_details,
- u"table_perf_comparison": table_perf_comparison,
- u"table_perf_comparison_nic": table_perf_comparison_nic,
- u"table_nics_comparison": table_nics_comparison,
u"table_soak_vs_ndr": table_soak_vs_ndr,
u"table_perf_trending_dash": table_perf_trending_dash,
u"table_perf_trending_dash_html": table_perf_trending_dash_html,
@@ -60,12 +59,16 @@ def generate_tables(spec, data):
u"table_failed_tests": table_failed_tests,
u"table_failed_tests_html": table_failed_tests_html,
u"table_oper_data_html": table_oper_data_html,
- u"table_comparison": table_comparison
+ u"table_comparison": table_comparison,
+ u"table_weekly_comparison": table_weekly_comparison,
+ u"table_job_spec_duration": table_job_spec_duration
}
logging.info(u"Generating the tables ...")
for table in spec.tables:
try:
+ if table[u"algorithm"] == u"table_weekly_comparison":
+ table[u"testbeds"] = spec.environment.get(u"testbeds", None)
generator[table[u"algorithm"]](table, data)
except NameError as err:
logging.error(
@@ -75,6 +78,96 @@ def generate_tables(spec, data):
logging.info(u"Done.")
+def table_job_spec_duration(table, input_data):
+ """Generate the table(s) with algorithm: table_job_spec_duration
+ specified in the specification file.
+
+ :param table: Table to generate.
+ :param input_data: Data to process.
+ :type table: pandas.Series
+ :type input_data: InputData
+ """
+
+ _ = input_data
+
+ logging.info(f" Generating the table {table.get(u'title', u'')} ...")
+
+ jb_type = table.get(u"jb-type", None)
+
+ tbl_lst = list()
+ if jb_type == u"iterative":
+ for line in table.get(u"lines", tuple()):
+ tbl_itm = {
+ u"name": line.get(u"job-spec", u""),
+ u"data": list()
+ }
+ for job, builds in line.get(u"data-set", dict()).items():
+ for build_nr in builds:
+ try:
+ minutes = input_data.metadata(
+ job, str(build_nr)
+ )[u"elapsedtime"] // 60000
+ except (KeyError, IndexError, ValueError, AttributeError):
+ continue
+ tbl_itm[u"data"].append(minutes)
+ tbl_itm[u"mean"] = mean(tbl_itm[u"data"])
+ tbl_itm[u"stdev"] = stdev(tbl_itm[u"data"])
+ tbl_lst.append(tbl_itm)
+ elif jb_type == u"coverage":
+ job = table.get(u"data", None)
+ if not job:
+ return
+ for line in table.get(u"lines", tuple()):
+ try:
+ tbl_itm = {
+ u"name": line.get(u"job-spec", u""),
+ u"mean": input_data.metadata(
+ list(job.keys())[0], str(line[u"build"])
+ )[u"elapsedtime"] // 60000,
+ u"stdev": float(u"nan")
+ }
+ tbl_itm[u"data"] = [tbl_itm[u"mean"], ]
+ except (KeyError, IndexError, ValueError, AttributeError):
+ continue
+ tbl_lst.append(tbl_itm)
+ else:
+ logging.warning(f"Wrong type of job-spec: {jb_type}. Skipping.")
+ return
+
+ for line in tbl_lst:
+ line[u"mean"] = \
+ f"{int(line[u'mean'] // 60):02d}:{int(line[u'mean'] % 60):02d}"
+ if math.isnan(line[u"stdev"]):
+ line[u"stdev"] = u""
+ else:
+ line[u"stdev"] = \
+ f"{int(line[u'stdev'] //60):02d}:{int(line[u'stdev'] % 60):02d}"
+
+ if not tbl_lst:
+ return
+
+ rows = list()
+ for itm in tbl_lst:
+ rows.append([
+ itm[u"name"],
+ f"{len(itm[u'data'])}",
+ f"{itm[u'mean']} +- {itm[u'stdev']}"
+ if itm[u"stdev"] != u"" else f"{itm[u'mean']}"
+ ])
+
+ txt_table = prettytable.PrettyTable(
+ [u"Job Specification", u"Nr of Runs", u"Duration [HH:MM]"]
+ )
+ for row in rows:
+ txt_table.add_row(row)
+ txt_table.align = u"r"
+ txt_table.align[u"Job Specification"] = u"l"
+
+ file_name = f"{table.get(u'output-file', u'')}.txt"
+ with open(file_name, u"wt", encoding='utf-8') as txt_file:
+ txt_file.write(str(txt_table))
+
+
def table_oper_data_html(table, input_data):
"""Generate the table(s) with algorithm: html_table_oper_data
specified in the specification file.
@@ -93,7 +186,7 @@ def table_oper_data_html(table, input_data):
)
data = input_data.filter_data(
table,
- params=[u"name", u"parent", u"show-run", u"type"],
+ params=[u"name", u"parent", u"telemetry-show-run", u"type"],
continue_on_error=True
)
if data.empty:
@@ -146,7 +239,8 @@ def table_oper_data_html(table, input_data):
)
thead.text = u"\t"
- if tst_data.get(u"show-run", u"No Data") == u"No Data":
+ if tst_data.get(u"telemetry-show-run", None) is None or \
+ isinstance(tst_data[u"telemetry-show-run"], str):
trow = ET.SubElement(
tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
)
@@ -176,17 +270,56 @@ def table_oper_data_html(table, input_data):
u"Average Vector Size"
)
- for dut_data in tst_data[u"show-run"].values():
+ for dut_data in tst_data[u"telemetry-show-run"].values():
trow = ET.SubElement(
tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
)
tcol = ET.SubElement(
trow, u"td", attrib=dict(align=u"left", colspan=u"6")
)
- if dut_data.get(u"threads", None) is None:
+ if dut_data.get(u"runtime", None) is None:
tcol.text = u"No Data"
continue
+ runtime = dict()
+ for item in dut_data[u"runtime"].get(u"data", tuple()):
+ tid = int(item[u"labels"][u"thread_id"])
+ if runtime.get(tid, None) is None:
+ runtime[tid] = dict()
+ gnode = item[u"labels"][u"graph_node"]
+ if runtime[tid].get(gnode, None) is None:
+ runtime[tid][gnode] = dict()
+ try:
+ runtime[tid][gnode][item[u"name"]] = float(item[u"value"])
+ except ValueError:
+ runtime[tid][gnode][item[u"name"]] = item[u"value"]
+
+ threads = dict({idx: list() for idx in range(len(runtime))})
+ for idx, run_data in runtime.items():
+ for gnode, gdata in run_data.items():
+ if gdata[u"vectors"] > 0:
+ clocks = gdata[u"clocks"] / gdata[u"vectors"]
+ elif gdata[u"calls"] > 0:
+ clocks = gdata[u"clocks"] / gdata[u"calls"]
+ elif gdata[u"suspends"] > 0:
+ clocks = gdata[u"clocks"] / gdata[u"suspends"]
+ else:
+ clocks = 0.0
+ if gdata[u"calls"] > 0:
+ vectors_call = gdata[u"vectors"] / gdata[u"calls"]
+ else:
+ vectors_call = 0.0
+ if int(gdata[u"calls"]) + int(gdata[u"vectors"]) + \
+ int(gdata[u"suspends"]):
+ threads[idx].append([
+ gnode,
+ int(gdata[u"calls"]),
+ int(gdata[u"vectors"]),
+ int(gdata[u"suspends"]),
+ clocks,
+ vectors_call
+ ])
+
bold = ET.SubElement(tcol, u"b")
bold.text = (
f"Host IP: {dut_data.get(u'host', '')}, "
@@ -200,7 +333,7 @@ def table_oper_data_html(table, input_data):
)
thead.text = u"\t"
- for thread_nr, thread in dut_data[u"threads"].items():
+ for thread_nr, thread in threads.items():
trow = ET.SubElement(
tbl, u"tr", attrib=dict(bgcolor=colors[u"header"])
)
@@ -323,7 +456,8 @@ def table_merged_details(table, input_data):
suite_name = suite[u"name"]
table_lst = list()
for test in data.keys():
- if data[test][u"parent"] not in suite_name:
+ if data[test][u"status"] != u"PASS" or \
+ data[test][u"parent"] not in suite_name:
continue
row_lst = list()
for column in table[u"columns"]:
@@ -348,13 +482,14 @@ def table_merged_details(table, input_data):
# Temporary solution: remove NDR results from message:
if bool(table.get(u'remove-ndr', False)):
try:
- col_data = col_data.split(u" |br| ", 1)[1]
+ col_data = col_data.split(u"\n", 1)[1]
except IndexError:
pass
+ col_data = col_data.replace(u'\n', u' |br| ').\
+ replace(u'\r', u'').replace(u'"', u"'")
col_data = f" |prein| {col_data} |preout| "
- elif column[u"data"].split(u" ")[1] in \
- (u"conf-history", u"show-run"):
- col_data = col_data.replace(u" |br| ", u"", 1)
+ elif column[u"data"].split(u" ")[1] in (u"conf-history", ):
+ col_data = col_data.replace(u'\n', u' |br| ')
col_data = f" |prein| {col_data[:-5]} |preout| "
row_lst.append(f'"{col_data}"')
except KeyError:
@@ -386,12 +521,7 @@ def _tpc_modify_test_name(test_name, ignore_nic=False):
:rtype: str
"""
test_name_mod = test_name.\
- replace(u"-ndrpdrdisc", u""). \
replace(u"-ndrpdr", u"").\
- replace(u"-pdrdisc", u""). \
- replace(u"-ndrdisc", u"").\
- replace(u"-pdr", u""). \
- replace(u"-ndr", u""). \
replace(u"1t1c", u"1c").\
replace(u"2t1c", u"1c"). \
replace(u"2t2c", u"2c").\
@@ -425,7 +555,7 @@ def _tpc_insert_data(target, src, include_tests):
"""Insert src data to the target structure.
:param target: Target structure where the data is placed.
- :param src: Source data to be placed into the target stucture.
+ :param src: Source data to be placed into the target structure.
:param include_tests: Which results will be included (MRR, NDR, PDR).
:type target: list
:type src: dict
@@ -433,64 +563,26 @@ def _tpc_insert_data(target, src, include_tests):
"""
try:
if include_tests == u"MRR":
- target.append(
- (
- src[u"result"][u"receive-rate"],
- src[u"result"][u"receive-stdev"]
- )
- )
+ target[u"mean"] = src[u"result"][u"receive-rate"]
+ target[u"stdev"] = src[u"result"][u"receive-stdev"]
elif include_tests == u"PDR":
- target.append(src[u"throughput"][u"PDR"][u"LOWER"])
+ target[u"data"].append(src[u"throughput"][u"PDR"][u"LOWER"])
elif include_tests == u"NDR":
- target.append(src[u"throughput"][u"NDR"][u"LOWER"])
+ target[u"data"].append(src[u"throughput"][u"NDR"][u"LOWER"])
+ elif u"latency" in include_tests:
+ keys = include_tests.split(u"-")
+ if len(keys) == 4:
+ lat = src[keys[0]][keys[1]][keys[2]][keys[3]]
+ target[u"data"].append(
+ float(u"nan") if lat == -1 else lat * 1e6
+ )
except (KeyError, TypeError):
pass
-def _tpc_sort_table(table):
- """Sort the table this way:
-
- 1. Put "New in CSIT-XXXX" at the first place.
- 2. Put "See footnote" at the second place.
- 3. Sort the rest by "Delta".
-
- :param table: Table to sort.
- :type table: list
- :returns: Sorted table.
- :rtype: list
- """
-
- tbl_new = list()
- tbl_see = list()
- tbl_delta = list()
- for item in table:
- if isinstance(item[-1], str):
- if u"New in CSIT" in item[-1]:
- tbl_new.append(item)
- elif u"See footnote" in item[-1]:
- tbl_see.append(item)
- else:
- tbl_delta.append(item)
-
- # Sort the tables:
- tbl_new.sort(key=lambda rel: rel[0], reverse=False)
- tbl_see.sort(key=lambda rel: rel[0], reverse=False)
- tbl_see.sort(key=lambda rel: rel[-2], reverse=False)
- tbl_delta.sort(key=lambda rel: rel[0], reverse=False)
- tbl_delta.sort(key=lambda rel: rel[-2], reverse=True)
-
- # Put the tables together:
- table = list()
- # We do not want "New in CSIT":
- # table.extend(tbl_new)
- table.extend(tbl_see)
- table.extend(tbl_delta)
-
- return table
-
-
def _tpc_generate_html_table(header, data, out_file_name, legend=u"",
- footnote=u"", sort_data=True, title=u""):
+ footnote=u"", sort_data=True, title=u"",
+ generate_rst=True):
"""Generate html table from input data with simple sorting possibility.
:param header: Table header.
@@ -504,6 +596,7 @@ def _tpc_generate_html_table(header, data, out_file_name, legend=u"",
:param footnote: The footnote to display below the table (and legend).
:param sort_data: If True the data sorting is enabled.
:param title: The table (and file) title.
+ :param generate_rst: If True, wrapping rst file is generated.
:type header: list
:type data: list of lists
:type out_file_name: str
@@ -511,6 +604,7 @@ def _tpc_generate_html_table(header, data, out_file_name, legend=u"",
:type footnote: str
:type sort_data: bool
:type title: str
+ :type generate_rst: bool
"""
try:
@@ -528,7 +622,7 @@ def _tpc_generate_html_table(header, data, out_file_name, legend=u"",
[u"left", u"left", u"right"],
[u"left", u"left", u"left", u"right"]
),
- u"width": ([28, 9], [4, 24, 10], [4, 4, 32, 10])
+ u"width": ([15, 9], [4, 24, 10], [4, 4, 32, 10])
}
df_data = pd.DataFrame(data, columns=header)
@@ -566,998 +660,112 @@ def _tpc_generate_html_table(header, data, out_file_name, legend=u"",
columnwidth=params[u"width"][idx],
header=table_header,
cells=dict(
- values=columns,
- fill_color=fill_color,
- align=params[u"align-itm"][idx],
- font=dict(
- family=u"Courier New",
- size=12
- )
- )
- )
- )
-
- buttons = list()
- menu_items = [f"{itm} (ascending)" for itm in header]
- menu_items_rev = [f"{itm} (descending)" for itm in header]
- menu_items.extend(menu_items_rev)
- for idx, hdr in enumerate(menu_items):
- visible = [False, ] * len(menu_items)
- visible[idx] = True
- buttons.append(
- dict(
- label=hdr.replace(u" [Mpps]", u""),
- method=u"update",
- args=[{u"visible": visible}],
- )
- )
-
- fig.update_layout(
- updatemenus=[
- go.layout.Updatemenu(
- type=u"dropdown",
- direction=u"down",
- x=0.0,
- xanchor=u"left",
- y=1.045,
- yanchor=u"top",
- active=len(menu_items) - 1,
- buttons=list(buttons)
- )
- ],
- )
- else:
- fig.add_trace(
- go.Table(
- columnwidth=params[u"width"][idx],
- header=table_header,
- cells=dict(
- values=[df_sorted.get(col) for col in header],
- fill_color=fill_color,
- align=params[u"align-itm"][idx],
- font=dict(
- family=u"Courier New",
- size=12
- )
- )
- )
- )
-
- ploff.plot(
- fig,
- show_link=False,
- auto_open=False,
- filename=f"{out_file_name}_in.html"
- )
-
- file_name = out_file_name.split(u"/")[-1]
- if u"vpp" in out_file_name:
- path = u"_tmp/src/vpp_performance_tests/comparisons/"
- else:
- path = u"_tmp/src/dpdk_performance_tests/comparisons/"
- with open(f"{path}{file_name}.rst", u"wt") as rst_file:
- rst_file.write(
- u"\n"
- u".. |br| raw:: html\n\n
\n\n\n"
- u".. |prein| raw:: html\n\n
\n\n\n" - u".. |preout| raw:: html\n\n\n\n" - ) - if title: - rst_file.write(f"{title}\n") - rst_file.write(f"{u'`' * len(title)}\n\n") - rst_file.write( - u".. raw:: html\n\n" - f' \n\n' - ) - if legend: - rst_file.write(legend[1:].replace(u"\n", u" |br| ")) - if footnote: - rst_file.write(footnote.replace(u"\n", u" |br| ")[1:]) - - -def table_perf_comparison(table, input_data): - """Generate the table(s) with algorithm: table_perf_comparison - specified in the specification file. - - :param table: Table to generate. - :param input_data: Data to process. - :type table: pandas.Series - :type input_data: InputData - """ - - logging.info(f" Generating the table {table.get(u'title', u'')} ...") - - # Transform the data - logging.info( - f" Creating the data set for the {table.get(u'type', u'')} " - f"{table.get(u'title', u'')}." - ) - data = input_data.filter_data(table, continue_on_error=True) - - # Prepare the header of the tables - try: - header = [u"Test Case", ] - legend = u"\nLegend:\n" - - rca_data = None - rca = table.get(u"rca", None) - if rca: - try: - with open(rca.get(u"data-file", u""), u"r") as rca_file: - rca_data = load(rca_file, Loader=FullLoader) - header.insert(0, rca.get(u"title", u"RCA")) - legend += ( - u"RCA: Reference to the Root Cause Analysis, see below.\n" - ) - except (YAMLError, IOError) as err: - logging.warning(repr(err)) - - history = table.get(u"history", list()) - for item in history: - header.extend( - [ - f"{item[u'title']} Avg({table[u'include-tests']})", - f"{item[u'title']} Stdev({table[u'include-tests']})" - ] - ) - legend += ( - f"{item[u'title']} Avg({table[u'include-tests']}): " - f"Mean value of {table[u'include-tests']} [Mpps] computed from " - f"a series of runs of the listed tests executed against " - f"{item[u'title']}.\n" - f"{item[u'title']} Stdev({table[u'include-tests']}): " - f"Standard deviation value of {table[u'include-tests']} [Mpps] " - f"computed from a series of runs of the listed tests executed " - f"against {item[u'title']}.\n" - ) - header.extend( - [ - f"{table[u'reference'][u'title']} " - f"Avg({table[u'include-tests']})", - f"{table[u'reference'][u'title']} " - f"Stdev({table[u'include-tests']})", - f"{table[u'compare'][u'title']} " - f"Avg({table[u'include-tests']})", - f"{table[u'compare'][u'title']} " - f"Stdev({table[u'include-tests']})", - f"Diff({table[u'reference'][u'title']}," - f"{table[u'compare'][u'title']})", - u"Stdev(Diff)" - ] - ) - header_str = u";".join(header) + u"\n" - legend += ( - f"{table[u'reference'][u'title']} " - f"Avg({table[u'include-tests']}): " - f"Mean value of {table[u'include-tests']} [Mpps] computed from a " - f"series of runs of the listed tests executed against " - f"{table[u'reference'][u'title']}.\n" - f"{table[u'reference'][u'title']} " - f"Stdev({table[u'include-tests']}): " - f"Standard deviation value of {table[u'include-tests']} [Mpps] " - f"computed from a series of runs of the listed tests executed " - f"against {table[u'reference'][u'title']}.\n" - f"{table[u'compare'][u'title']} " - f"Avg({table[u'include-tests']}): " - f"Mean value of {table[u'include-tests']} [Mpps] computed from a " - f"series of runs of the listed tests executed against " - f"{table[u'compare'][u'title']}.\n" - f"{table[u'compare'][u'title']} " - f"Stdev({table[u'include-tests']}): " - f"Standard deviation value of {table[u'include-tests']} [Mpps] " - f"computed from a series of runs of the listed tests executed " - f"against {table[u'compare'][u'title']}.\n" - f"Diff({table[u'reference'][u'title']}," - f"{table[u'compare'][u'title']}): " - f"Percentage change calculated for mean values.\n" - u"Stdev(Diff): " - u"Standard deviation of percentage change calculated for mean " - u"values.\n" - u"NT: Not Tested\n" - ) - except (AttributeError, KeyError) as err: - logging.error(f"The model is invalid, missing parameter: {repr(err)}") - return - - # Prepare data to the table: - tbl_dict = dict() - for job, builds in table[u"reference"][u"data"].items(): - for build in builds: - for tst_name, tst_data in data[job][str(build)].items(): - tst_name_mod = _tpc_modify_test_name(tst_name) - if (u"across topologies" in table[u"title"].lower() or - (u" 3n-" in table[u"title"].lower() and - u" 2n-" in table[u"title"].lower())): - tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") - if tbl_dict.get(tst_name_mod, None) is None: - name = tst_data[u'name'].rsplit(u'-', 1)[0] - if u"across testbeds" in table[u"title"].lower() or \ - u"across topologies" in table[u"title"].lower(): - name = _tpc_modify_displayed_test_name(name) - tbl_dict[tst_name_mod] = { - u"name": name, - u"replace-ref": True, - u"replace-cmp": True, - u"ref-data": list(), - u"cmp-data": list() - } - _tpc_insert_data(target=tbl_dict[tst_name_mod][u"ref-data"], - src=tst_data, - include_tests=table[u"include-tests"]) - - replacement = table[u"reference"].get(u"data-replacement", None) - if replacement: - rpl_data = input_data.filter_data( - table, data=replacement, continue_on_error=True) - for job, builds in replacement.items(): - for build in builds: - for tst_name, tst_data in rpl_data[job][str(build)].items(): - tst_name_mod = _tpc_modify_test_name(tst_name) - if (u"across topologies" in table[u"title"].lower() or - (u" 3n-" in table[u"title"].lower() and - u" 2n-" in table[u"title"].lower())): - tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") - if tbl_dict.get(tst_name_mod, None) is None: - name = tst_data[u'name'].rsplit(u'-', 1)[0] - if u"across testbeds" in table[u"title"].lower() or \ - u"across topologies" in table[u"title"].lower(): - name = _tpc_modify_displayed_test_name(name) - tbl_dict[tst_name_mod] = { - u"name": name, - u"replace-ref": False, - u"replace-cmp": True, - u"ref-data": list(), - u"cmp-data": list() - } - if tbl_dict[tst_name_mod][u"replace-ref"]: - tbl_dict[tst_name_mod][u"replace-ref"] = False - tbl_dict[tst_name_mod][u"ref-data"] = list() - - _tpc_insert_data( - target=tbl_dict[tst_name_mod][u"ref-data"], - src=tst_data, - include_tests=table[u"include-tests"] - ) - - for job, builds in table[u"compare"][u"data"].items(): - for build in builds: - for tst_name, tst_data in data[job][str(build)].items(): - tst_name_mod = _tpc_modify_test_name(tst_name) - if (u"across topologies" in table[u"title"].lower() or - (u" 3n-" in table[u"title"].lower() and - u" 2n-" in table[u"title"].lower())): - tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") - if tbl_dict.get(tst_name_mod, None) is None: - name = tst_data[u'name'].rsplit(u'-', 1)[0] - if u"across testbeds" in table[u"title"].lower() or \ - u"across topologies" in table[u"title"].lower(): - name = _tpc_modify_displayed_test_name(name) - tbl_dict[tst_name_mod] = { - u"name": name, - u"replace-ref": False, - u"replace-cmp": True, - u"ref-data": list(), - u"cmp-data": list() - } - _tpc_insert_data( - target=tbl_dict[tst_name_mod][u"cmp-data"], - src=tst_data, - include_tests=table[u"include-tests"] - ) - - replacement = table[u"compare"].get(u"data-replacement", None) - if replacement: - rpl_data = input_data.filter_data( - table, data=replacement, continue_on_error=True) - for job, builds in replacement.items(): - for build in builds: - for tst_name, tst_data in rpl_data[job][str(build)].items(): - tst_name_mod = _tpc_modify_test_name(tst_name) - if (u"across topologies" in table[u"title"].lower() or - (u" 3n-" in table[u"title"].lower() and - u" 2n-" in table[u"title"].lower())): - tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") - if tbl_dict.get(tst_name_mod, None) is None: - name = tst_data[u'name'].rsplit(u'-', 1)[0] - if u"across testbeds" in table[u"title"].lower() or \ - u"across topologies" in table[u"title"].lower(): - name = _tpc_modify_displayed_test_name(name) - tbl_dict[tst_name_mod] = { - u"name": name, - u"replace-ref": False, - u"replace-cmp": False, - u"ref-data": list(), - u"cmp-data": list() - } - if tbl_dict[tst_name_mod][u"replace-cmp"]: - tbl_dict[tst_name_mod][u"replace-cmp"] = False - tbl_dict[tst_name_mod][u"cmp-data"] = list() - - _tpc_insert_data( - target=tbl_dict[tst_name_mod][u"cmp-data"], - src=tst_data, - include_tests=table[u"include-tests"] - ) - - for item in history: - for job, builds in item[u"data"].items(): - for build in builds: - for tst_name, tst_data in data[job][str(build)].items(): - tst_name_mod = _tpc_modify_test_name(tst_name) - if (u"across topologies" in table[u"title"].lower() or - (u" 3n-" in table[u"title"].lower() and - u" 2n-" in table[u"title"].lower())): - tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") - if tbl_dict.get(tst_name_mod, None) is None: - continue - if tbl_dict[tst_name_mod].get(u"history", None) is None: - tbl_dict[tst_name_mod][u"history"] = OrderedDict() - if tbl_dict[tst_name_mod][u"history"].\ - get(item[u"title"], None) is None: - tbl_dict[tst_name_mod][u"history"][item[ - u"title"]] = list() - try: - if table[u"include-tests"] == u"MRR": - res = (tst_data[u"result"][u"receive-rate"], - tst_data[u"result"][u"receive-stdev"]) - elif table[u"include-tests"] == u"PDR": - res = tst_data[u"throughput"][u"PDR"][u"LOWER"] - elif table[u"include-tests"] == u"NDR": - res = tst_data[u"throughput"][u"NDR"][u"LOWER"] - else: - continue - tbl_dict[tst_name_mod][u"history"][item[u"title"]].\ - append(res) - except (TypeError, KeyError): - pass - - tbl_lst = list() - for tst_name in tbl_dict: - item = [tbl_dict[tst_name][u"name"], ] - if history: - if tbl_dict[tst_name].get(u"history", None) is not None: - for hist_data in tbl_dict[tst_name][u"history"].values(): - if hist_data: - if table[u"include-tests"] == u"MRR": - item.append(round(hist_data[0][0] / 1e6, 1)) - item.append(round(hist_data[0][1] / 1e6, 1)) - else: - item.append(round(mean(hist_data) / 1e6, 1)) - item.append(round(stdev(hist_data) / 1e6, 1)) - else: - item.extend([u"NT", u"NT"]) - else: - item.extend([u"NT", u"NT"]) - data_r = tbl_dict[tst_name][u"ref-data"] - if data_r: - if table[u"include-tests"] == u"MRR": - data_r_mean = data_r[0][0] - data_r_stdev = data_r[0][1] - else: - data_r_mean = mean(data_r) - data_r_stdev = stdev(data_r) - item.append(round(data_r_mean / 1e6, 1)) - item.append(round(data_r_stdev / 1e6, 1)) - else: - data_r_mean = None - data_r_stdev = None - item.extend([u"NT", u"NT"]) - data_c = tbl_dict[tst_name][u"cmp-data"] - if data_c: - if table[u"include-tests"] == u"MRR": - data_c_mean = data_c[0][0] - data_c_stdev = data_c[0][1] - else: - data_c_mean = mean(data_c) - data_c_stdev = stdev(data_c) - item.append(round(data_c_mean / 1e6, 1)) - item.append(round(data_c_stdev / 1e6, 1)) - else: - data_c_mean = None - data_c_stdev = None - item.extend([u"NT", u"NT"]) - if item[-2] == u"NT": - pass - elif item[-4] == u"NT": - item.append(u"New in CSIT-2001") - item.append(u"New in CSIT-2001") - elif data_r_mean is not None and data_c_mean is not None: - delta, d_stdev = relative_change_stdev( - data_r_mean, data_c_mean, data_r_stdev, data_c_stdev - ) - try: - item.append(round(delta)) - except ValueError: - item.append(delta) - try: - item.append(round(d_stdev)) - except ValueError: - item.append(d_stdev) - if rca_data: - rca_nr = rca_data.get(item[0], u"-") - item.insert(0, f"[{rca_nr}]" if rca_nr != u"-" else u"-") - if (len(item) == len(header)) and (item[-4] != u"NT"): - tbl_lst.append(item) - - tbl_lst = _tpc_sort_table(tbl_lst) - - # Generate csv tables: - csv_file = f"{table[u'output-file']}.csv" - with open(csv_file, u"wt") as file_handler: - file_handler.write(header_str) - for test in tbl_lst: - file_handler.write(u";".join([str(item) for item in test]) + u"\n") - - txt_file_name = f"{table[u'output-file']}.txt" - convert_csv_to_pretty_txt(csv_file, txt_file_name, delimiter=u";") - - footnote = u"" - with open(txt_file_name, u'a') as txt_file: - txt_file.write(legend) - if rca_data: - footnote = rca_data.get(u"footnote", u"") - if footnote: - txt_file.write(footnote) - txt_file.write(u":END") - - # Generate html table: - _tpc_generate_html_table( - header, - tbl_lst, - table[u'output-file'], - legend=legend, - footnote=footnote, - title=table.get(u"title", u"") - ) - - -def table_perf_comparison_nic(table, input_data): - """Generate the table(s) with algorithm: table_perf_comparison - specified in the specification file. - - :param table: Table to generate. - :param input_data: Data to process. - :type table: pandas.Series - :type input_data: InputData - """ - - logging.info(f" Generating the table {table.get(u'title', u'')} ...") - - # Transform the data - logging.info( - f" Creating the data set for the {table.get(u'type', u'')} " - f"{table.get(u'title', u'')}." - ) - data = input_data.filter_data(table, continue_on_error=True) - - # Prepare the header of the tables - try: - header = [u"Test Case", ] - legend = u"\nLegend:\n" - - rca_data = None - rca = table.get(u"rca", None) - if rca: - try: - with open(rca.get(u"data-file", ""), u"r") as rca_file: - rca_data = load(rca_file, Loader=FullLoader) - header.insert(0, rca.get(u"title", "RCA")) - legend += ( - u"RCA: Reference to the Root Cause Analysis, see below.\n" - ) - except (YAMLError, IOError) as err: - logging.warning(repr(err)) - - history = table.get(u"history", list()) - for item in history: - header.extend( - [ - f"{item[u'title']} Avg({table[u'include-tests']})", - f"{item[u'title']} Stdev({table[u'include-tests']})" - ] - ) - legend += ( - f"{item[u'title']} Avg({table[u'include-tests']}): " - f"Mean value of {table[u'include-tests']} [Mpps] computed from " - f"a series of runs of the listed tests executed against " - f"{item[u'title']}.\n" - f"{item[u'title']} Stdev({table[u'include-tests']}): " - f"Standard deviation value of {table[u'include-tests']} [Mpps] " - f"computed from a series of runs of the listed tests executed " - f"against {item[u'title']}.\n" - ) - header.extend( - [ - f"{table[u'reference'][u'title']} " - f"Avg({table[u'include-tests']})", - f"{table[u'reference'][u'title']} " - f"Stdev({table[u'include-tests']})", - f"{table[u'compare'][u'title']} " - f"Avg({table[u'include-tests']})", - f"{table[u'compare'][u'title']} " - f"Stdev({table[u'include-tests']})", - f"Diff({table[u'reference'][u'title']}," - f"{table[u'compare'][u'title']})", - u"Stdev(Diff)" - ] - ) - header_str = u";".join(header) + u"\n" - legend += ( - f"{table[u'reference'][u'title']} " - f"Avg({table[u'include-tests']}): " - f"Mean value of {table[u'include-tests']} [Mpps] computed from a " - f"series of runs of the listed tests executed against " - f"{table[u'reference'][u'title']}.\n" - f"{table[u'reference'][u'title']} " - f"Stdev({table[u'include-tests']}): " - f"Standard deviation value of {table[u'include-tests']} [Mpps] " - f"computed from a series of runs of the listed tests executed " - f"against {table[u'reference'][u'title']}.\n" - f"{table[u'compare'][u'title']} " - f"Avg({table[u'include-tests']}): " - f"Mean value of {table[u'include-tests']} [Mpps] computed from a " - f"series of runs of the listed tests executed against " - f"{table[u'compare'][u'title']}.\n" - f"{table[u'compare'][u'title']} " - f"Stdev({table[u'include-tests']}): " - f"Standard deviation value of {table[u'include-tests']} [Mpps] " - f"computed from a series of runs of the listed tests executed " - f"against {table[u'compare'][u'title']}.\n" - f"Diff({table[u'reference'][u'title']}," - f"{table[u'compare'][u'title']}): " - f"Percentage change calculated for mean values.\n" - u"Stdev(Diff): " - u"Standard deviation of percentage change calculated for mean " - u"values.\n" - u"NT: Not Tested\n" - ) - except (AttributeError, KeyError) as err: - logging.error(f"The model is invalid, missing parameter: {repr(err)}") - return - - # Prepare data to the table: - tbl_dict = dict() - for job, builds in table[u"reference"][u"data"].items(): - for build in builds: - for tst_name, tst_data in data[job][str(build)].items(): - if table[u"reference"][u"nic"] not in tst_data[u"tags"]: - continue - tst_name_mod = _tpc_modify_test_name(tst_name, ignore_nic=True) - if (u"across topologies" in table[u"title"].lower() or - (u" 3n-" in table[u"title"].lower() and - u" 2n-" in table[u"title"].lower())): - tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") - if tbl_dict.get(tst_name_mod, None) is None: - name = tst_data[u'name'].rsplit(u'-', 1)[0] - if u"across testbeds" in table[u"title"].lower() or \ - u"across topologies" in table[u"title"].lower(): - name = _tpc_modify_displayed_test_name(name) - tbl_dict[tst_name_mod] = { - u"name": name, - u"replace-ref": True, - u"replace-cmp": True, - u"ref-data": list(), - u"cmp-data": list() - } - _tpc_insert_data( - target=tbl_dict[tst_name_mod][u"ref-data"], - src=tst_data, - include_tests=table[u"include-tests"] - ) - - replacement = table[u"reference"].get(u"data-replacement", None) - if replacement: - rpl_data = input_data.filter_data( - table, data=replacement, continue_on_error=True) - for job, builds in replacement.items(): - for build in builds: - for tst_name, tst_data in rpl_data[job][str(build)].items(): - if table[u"reference"][u"nic"] not in tst_data[u"tags"]: - continue - tst_name_mod = \ - _tpc_modify_test_name(tst_name, ignore_nic=True) - if (u"across topologies" in table[u"title"].lower() or - (u" 3n-" in table[u"title"].lower() and - u" 2n-" in table[u"title"].lower())): - tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") - if tbl_dict.get(tst_name_mod, None) is None: - name = tst_data[u'name'].rsplit(u'-', 1)[0] - if u"across testbeds" in table[u"title"].lower() or \ - u"across topologies" in table[u"title"].lower(): - name = _tpc_modify_displayed_test_name(name) - tbl_dict[tst_name_mod] = { - u"name": name, - u"replace-ref": False, - u"replace-cmp": True, - u"ref-data": list(), - u"cmp-data": list() - } - if tbl_dict[tst_name_mod][u"replace-ref"]: - tbl_dict[tst_name_mod][u"replace-ref"] = False - tbl_dict[tst_name_mod][u"ref-data"] = list() - - _tpc_insert_data( - target=tbl_dict[tst_name_mod][u"ref-data"], - src=tst_data, - include_tests=table[u"include-tests"] - ) - - for job, builds in table[u"compare"][u"data"].items(): - for build in builds: - for tst_name, tst_data in data[job][str(build)].items(): - if table[u"compare"][u"nic"] not in tst_data[u"tags"]: - continue - tst_name_mod = _tpc_modify_test_name(tst_name, ignore_nic=True) - if (u"across topologies" in table[u"title"].lower() or - (u" 3n-" in table[u"title"].lower() and - u" 2n-" in table[u"title"].lower())): - tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") - if tbl_dict.get(tst_name_mod, None) is None: - name = tst_data[u'name'].rsplit(u'-', 1)[0] - if u"across testbeds" in table[u"title"].lower() or \ - u"across topologies" in table[u"title"].lower(): - name = _tpc_modify_displayed_test_name(name) - tbl_dict[tst_name_mod] = { - u"name": name, - u"replace-ref": False, - u"replace-cmp": True, - u"ref-data": list(), - u"cmp-data": list() - } - _tpc_insert_data( - target=tbl_dict[tst_name_mod][u"cmp-data"], - src=tst_data, - include_tests=table[u"include-tests"] - ) - - replacement = table[u"compare"].get(u"data-replacement", None) - if replacement: - rpl_data = input_data.filter_data( - table, data=replacement, continue_on_error=True) - for job, builds in replacement.items(): - for build in builds: - for tst_name, tst_data in rpl_data[job][str(build)].items(): - if table[u"compare"][u"nic"] not in tst_data[u"tags"]: - continue - tst_name_mod = \ - _tpc_modify_test_name(tst_name, ignore_nic=True) - if (u"across topologies" in table[u"title"].lower() or - (u" 3n-" in table[u"title"].lower() and - u" 2n-" in table[u"title"].lower())): - tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") - if tbl_dict.get(tst_name_mod, None) is None: - name = tst_data[u'name'].rsplit(u'-', 1)[0] - if u"across testbeds" in table[u"title"].lower() or \ - u"across topologies" in table[u"title"].lower(): - name = _tpc_modify_displayed_test_name(name) - tbl_dict[tst_name_mod] = { - u"name": name, - u"replace-ref": False, - u"replace-cmp": False, - u"ref-data": list(), - u"cmp-data": list() - } - if tbl_dict[tst_name_mod][u"replace-cmp"]: - tbl_dict[tst_name_mod][u"replace-cmp"] = False - tbl_dict[tst_name_mod][u"cmp-data"] = list() - - _tpc_insert_data( - target=tbl_dict[tst_name_mod][u"cmp-data"], - src=tst_data, - include_tests=table[u"include-tests"] - ) - - for item in history: - for job, builds in item[u"data"].items(): - for build in builds: - for tst_name, tst_data in data[job][str(build)].items(): - if item[u"nic"] not in tst_data[u"tags"]: - continue - tst_name_mod = \ - _tpc_modify_test_name(tst_name, ignore_nic=True) - if (u"across topologies" in table[u"title"].lower() or - (u" 3n-" in table[u"title"].lower() and - u" 2n-" in table[u"title"].lower())): - tst_name_mod = tst_name_mod.replace(u"2n1l-", u"") - if tbl_dict.get(tst_name_mod, None) is None: - continue - if tbl_dict[tst_name_mod].get(u"history", None) is None: - tbl_dict[tst_name_mod][u"history"] = OrderedDict() - if tbl_dict[tst_name_mod][u"history"].\ - get(item[u"title"], None) is None: - tbl_dict[tst_name_mod][u"history"][item[ - u"title"]] = list() - try: - if table[u"include-tests"] == u"MRR": - res = (tst_data[u"result"][u"receive-rate"], - tst_data[u"result"][u"receive-stdev"]) - elif table[u"include-tests"] == u"PDR": - res = tst_data[u"throughput"][u"PDR"][u"LOWER"] - elif table[u"include-tests"] == u"NDR": - res = tst_data[u"throughput"][u"NDR"][u"LOWER"] - else: - continue - tbl_dict[tst_name_mod][u"history"][item[u"title"]].\ - append(res) - except (TypeError, KeyError): - pass - - tbl_lst = list() - for tst_name in tbl_dict: - item = [tbl_dict[tst_name][u"name"], ] - if history: - if tbl_dict[tst_name].get(u"history", None) is not None: - for hist_data in tbl_dict[tst_name][u"history"].values(): - if hist_data: - if table[u"include-tests"] == u"MRR": - item.append(round(hist_data[0][0] / 1e6, 1)) - item.append(round(hist_data[0][1] / 1e6, 1)) - else: - item.append(round(mean(hist_data) / 1e6, 1)) - item.append(round(stdev(hist_data) / 1e6, 1)) - else: - item.extend([u"NT", u"NT"]) - else: - item.extend([u"NT", u"NT"]) - data_r = tbl_dict[tst_name][u"ref-data"] - if data_r: - if table[u"include-tests"] == u"MRR": - data_r_mean = data_r[0][0] - data_r_stdev = data_r[0][1] - else: - data_r_mean = mean(data_r) - data_r_stdev = stdev(data_r) - item.append(round(data_r_mean / 1e6, 1)) - item.append(round(data_r_stdev / 1e6, 1)) - else: - data_r_mean = None - data_r_stdev = None - item.extend([u"NT", u"NT"]) - data_c = tbl_dict[tst_name][u"cmp-data"] - if data_c: - if table[u"include-tests"] == u"MRR": - data_c_mean = data_c[0][0] - data_c_stdev = data_c[0][1] - else: - data_c_mean = mean(data_c) - data_c_stdev = stdev(data_c) - item.append(round(data_c_mean / 1e6, 1)) - item.append(round(data_c_stdev / 1e6, 1)) - else: - data_c_mean = None - data_c_stdev = None - item.extend([u"NT", u"NT"]) - if item[-2] == u"NT": - pass - elif item[-4] == u"NT": - item.append(u"New in CSIT-2001") - item.append(u"New in CSIT-2001") - elif data_r_mean is not None and data_c_mean is not None: - delta, d_stdev = relative_change_stdev( - data_r_mean, data_c_mean, data_r_stdev, data_c_stdev - ) - try: - item.append(round(delta)) - except ValueError: - item.append(delta) - try: - item.append(round(d_stdev)) - except ValueError: - item.append(d_stdev) - if rca_data: - rca_nr = rca_data.get(item[0], u"-") - item.insert(0, f"[{rca_nr}]" if rca_nr != u"-" else u"-") - if (len(item) == len(header)) and (item[-4] != u"NT"): - tbl_lst.append(item) - - tbl_lst = _tpc_sort_table(tbl_lst) - - # Generate csv tables: - csv_file = f"{table[u'output-file']}.csv" - with open(csv_file, u"wt") as file_handler: - file_handler.write(header_str) - for test in tbl_lst: - file_handler.write(u";".join([str(item) for item in test]) + u"\n") - - txt_file_name = f"{table[u'output-file']}.txt" - convert_csv_to_pretty_txt(csv_file, txt_file_name, delimiter=u";") - - footnote = u"" - with open(txt_file_name, u'a') as txt_file: - txt_file.write(legend) - if rca_data: - footnote = rca_data.get(u"footnote", u"") - if footnote: - txt_file.write(footnote) - txt_file.write(u":END") - - # Generate html table: - _tpc_generate_html_table( - header, - tbl_lst, - table[u'output-file'], - legend=legend, - footnote=footnote, - title=table.get(u"title", u"") - ) - - -def table_nics_comparison(table, input_data): - """Generate the table(s) with algorithm: table_nics_comparison - specified in the specification file. - - :param table: Table to generate. - :param input_data: Data to process. - :type table: pandas.Series - :type input_data: InputData - """ - - logging.info(f" Generating the table {table.get(u'title', u'')} ...") - - # Transform the data - logging.info( - f" Creating the data set for the {table.get(u'type', u'')} " - f"{table.get(u'title', u'')}." - ) - data = input_data.filter_data(table, continue_on_error=True) - - # Prepare the header of the tables - try: - header = [ - u"Test Case", - f"{table[u'reference'][u'title']} " - f"Avg({table[u'include-tests']})", - f"{table[u'reference'][u'title']} " - f"Stdev({table[u'include-tests']})", - f"{table[u'compare'][u'title']} " - f"Avg({table[u'include-tests']})", - f"{table[u'compare'][u'title']} " - f"Stdev({table[u'include-tests']})", - f"Diff({table[u'reference'][u'title']}," - f"{table[u'compare'][u'title']})", - u"Stdev(Diff)" - ] - legend = ( - u"\nLegend:\n" - f"{table[u'reference'][u'title']} " - f"Avg({table[u'include-tests']}): " - f"Mean value of {table[u'include-tests']} [Mpps] computed from a " - f"series of runs of the listed tests executed using " - f"{table[u'reference'][u'title']} NIC.\n" - f"{table[u'reference'][u'title']} " - f"Stdev({table[u'include-tests']}): " - f"Standard deviation value of {table[u'include-tests']} [Mpps] " - f"computed from a series of runs of the listed tests executed " - f"using {table[u'reference'][u'title']} NIC.\n" - f"{table[u'compare'][u'title']} " - f"Avg({table[u'include-tests']}): " - f"Mean value of {table[u'include-tests']} [Mpps] computed from a " - f"series of runs of the listed tests executed using " - f"{table[u'compare'][u'title']} NIC.\n" - f"{table[u'compare'][u'title']} " - f"Stdev({table[u'include-tests']}): " - f"Standard deviation value of {table[u'include-tests']} [Mpps] " - f"computed from a series of runs of the listed tests executed " - f"using {table[u'compare'][u'title']} NIC.\n" - f"Diff({table[u'reference'][u'title']}," - f"{table[u'compare'][u'title']}): " - f"Percentage change calculated for mean values.\n" - u"Stdev(Diff): " - u"Standard deviation of percentage change calculated for mean " - u"values.\n" - u":END" - ) - - except (AttributeError, KeyError) as err: - logging.error(f"The model is invalid, missing parameter: {repr(err)}") - return - - # Prepare data to the table: - tbl_dict = dict() - for job, builds in table[u"data"].items(): - for build in builds: - for tst_name, tst_data in data[job][str(build)].items(): - tst_name_mod = _tpc_modify_test_name(tst_name, ignore_nic=True) - if tbl_dict.get(tst_name_mod, None) is None: - name = tst_data[u'name'].rsplit(u'-', 1)[0] - tbl_dict[tst_name_mod] = { - u"name": name, - u"ref-data": list(), - u"cmp-data": list() - } - try: - if table[u"include-tests"] == u"MRR": - result = (tst_data[u"result"][u"receive-rate"], - tst_data[u"result"][u"receive-stdev"]) - elif table[u"include-tests"] == u"PDR": - result = tst_data[u"throughput"][u"PDR"][u"LOWER"] - elif table[u"include-tests"] == u"NDR": - result = tst_data[u"throughput"][u"NDR"][u"LOWER"] - else: - continue - - if result and \ - table[u"reference"][u"nic"] in tst_data[u"tags"]: - tbl_dict[tst_name_mod][u"ref-data"].append(result) - elif result and \ - table[u"compare"][u"nic"] in tst_data[u"tags"]: - tbl_dict[tst_name_mod][u"cmp-data"].append(result) - except (TypeError, KeyError) as err: - logging.debug(f"No data for {tst_name}\n{repr(err)}") - # No data in output.xml for this test - - tbl_lst = list() - for tst_name in tbl_dict: - item = [tbl_dict[tst_name][u"name"], ] - data_r = tbl_dict[tst_name][u"ref-data"] - if data_r: - if table[u"include-tests"] == u"MRR": - data_r_mean = data_r[0][0] - data_r_stdev = data_r[0][1] - else: - data_r_mean = mean(data_r) - data_r_stdev = stdev(data_r) - item.append(round(data_r_mean / 1e6, 1)) - item.append(round(data_r_stdev / 1e6, 1)) - else: - data_r_mean = None - data_r_stdev = None - item.extend([None, None]) - data_c = tbl_dict[tst_name][u"cmp-data"] - if data_c: - if table[u"include-tests"] == u"MRR": - data_c_mean = data_c[0][0] - data_c_stdev = data_c[0][1] - else: - data_c_mean = mean(data_c) - data_c_stdev = stdev(data_c) - item.append(round(data_c_mean / 1e6, 1)) - item.append(round(data_c_stdev / 1e6, 1)) - else: - data_c_mean = None - data_c_stdev = None - item.extend([None, None]) - if data_r_mean is not None and data_c_mean is not None: - delta, d_stdev = relative_change_stdev( - data_r_mean, data_c_mean, data_r_stdev, data_c_stdev + values=columns, + fill_color=fill_color, + align=params[u"align-itm"][idx], + font=dict( + family=u"Courier New", + size=12 + ) + ) + ) ) - try: - item.append(round(delta)) - except ValueError: - item.append(delta) - try: - item.append(round(d_stdev)) - except ValueError: - item.append(d_stdev) - tbl_lst.append(item) - # Sort the table according to the relative change - tbl_lst.sort(key=lambda rel: rel[-1], reverse=True) + buttons = list() + menu_items = [f"{itm} (ascending)" for itm in header] + menu_items.extend([f"{itm} (descending)" for itm in header]) + for idx, hdr in enumerate(menu_items): + visible = [False, ] * len(menu_items) + visible[idx] = True + buttons.append( + dict( + label=hdr.replace(u" [Mpps]", u""), + method=u"update", + args=[{u"visible": visible}], + ) + ) - # Generate csv tables: - with open(f"{table[u'output-file']}.csv", u"wt") as file_handler: - file_handler.write(u";".join(header) + u"\n") - for test in tbl_lst: - file_handler.write(u";".join([str(item) for item in test]) + u"\n") + fig.update_layout( + updatemenus=[ + go.layout.Updatemenu( + type=u"dropdown", + direction=u"down", + x=0.0, + xanchor=u"left", + y=1.002, + yanchor=u"bottom", + active=len(menu_items) - 1, + buttons=list(buttons) + ) + ], + ) + else: + fig.add_trace( + go.Table( + columnwidth=params[u"width"][idx], + header=table_header, + cells=dict( + values=[df_sorted.get(col) for col in header], + fill_color=fill_color, + align=params[u"align-itm"][idx], + font=dict( + family=u"Courier New", + size=12 + ) + ) + ) + ) - convert_csv_to_pretty_txt(f"{table[u'output-file']}.csv", - f"{table[u'output-file']}.txt", - delimiter=u";") + ploff.plot( + fig, + show_link=False, + auto_open=False, + filename=f"{out_file_name}_in.html" + ) - with open(table[u'output-file'], u'a') as txt_file: - txt_file.write(legend) + if not generate_rst: + return - # Generate html table: - _tpc_generate_html_table( - header, - tbl_lst, - table[u'output-file'], - legend=legend, - title=table.get(u"title", u"") - ) + file_name = out_file_name.split(u"/")[-1] + if u"vpp" in out_file_name: + path = u"_tmp/src/vpp_performance_tests/comparisons/" + else: + path = u"_tmp/src/dpdk_performance_tests/comparisons/" + logging.info(f" Writing the HTML file to {path}{file_name}.rst") + with open(f"{path}{file_name}.rst", u"wt") as rst_file: + rst_file.write( + u"\n" + u".. |br| raw:: html\n\n
\n\n\n" + u".. |preout| raw:: html\n\n\n\n" + ) + if title: + rst_file.write(f"{title}\n") + rst_file.write(f"{u'`' * len(title)}\n\n") + rst_file.write( + u".. raw:: html\n\n" + f' \n\n' + ) + + if legend: + try: + itm_lst = legend[1:-2].split(u"\n") + rst_file.write( + f"{itm_lst[0]}\n\n- " + u'\n- '.join(itm_lst[1:]) + u"\n\n" + ) + except IndexError as err: + logging.error(f"Legend cannot be written to html file\n{err}") + if footnote: + try: + itm_lst = footnote[1:].split(u"\n") + rst_file.write( + f"{itm_lst[0]}\n\n- " + u'\n- '.join(itm_lst[1:]) + u"\n\n" + ) + except IndexError as err: + logging.error(f"Footnote cannot be written to html file\n{err}") def table_soak_vs_ndr(table, input_data): @@ -1610,8 +818,7 @@ def table_soak_vs_ndr(table, input_data): f"Percentage change calculated for mean values.\n" u"Stdev(Diff): " u"Standard deviation of percentage change calculated for mean " - u"values.\n" - u":END" + u"values." ) except (AttributeError, KeyError) as err: logging.error(f"The model is invalid, missing parameter: {repr(err)}") @@ -1719,17 +926,17 @@ def table_soak_vs_ndr(table, input_data): tbl_lst.sort(key=lambda rel: rel[-1], reverse=True) # Generate csv tables: - csv_file = f"{table[u'output-file']}.csv" - with open(csv_file, u"wt") as file_handler: + csv_file_name = f"{table[u'output-file']}.csv" + with open(csv_file_name, u"wt") as file_handler: file_handler.write(header_str) for test in tbl_lst: file_handler.write(u";".join([str(item) for item in test]) + u"\n") convert_csv_to_pretty_txt( - csv_file, f"{table[u'output-file']}.txt", delimiter=u";" + csv_file_name, f"{table[u'output-file']}.txt", delimiter=u";" ) - with open(f"{table[u'output-file']}.txt", u'a') as txt_file: - txt_file.write(legend) + with open(f"{table[u'output-file']}.txt", u'a') as file_handler: + file_handler.write(legend) # Generate html table: _tpc_generate_html_table( @@ -1765,13 +972,15 @@ def table_perf_trending_dash(table, input_data): header = [ u"Test Case", u"Trend [Mpps]", - u"Short-Term Change [%]", - u"Long-Term Change [%]", + u"Number of runs [#]", + u"Trend Change [%]", u"Regressions [#]", u"Progressions [#]" ] header_str = u",".join(header) + u"\n" + incl_tests = table.get(u"include-tests", u"MRR") + # Prepare data to the table: tbl_dict = dict() for job, builds in table[u"data"].items(): @@ -1789,8 +998,15 @@ def table_perf_trending_dash(table, input_data): u"data": OrderedDict() } try: - tbl_dict[tst_name][u"data"][str(build)] = \ - tst_data[u"result"][u"receive-rate"] + if incl_tests == u"MRR": + tbl_dict[tst_name][u"data"][str(build)] = \ + tst_data[u"result"][u"receive-rate"] + elif incl_tests == u"NDR": + tbl_dict[tst_name][u"data"][str(build)] = \ + tst_data[u"throughput"][u"NDR"][u"LOWER"] + elif incl_tests == u"PDR": + tbl_dict[tst_name][u"data"][str(build)] = \ + tst_data[u"throughput"][u"PDR"][u"LOWER"] except (TypeError, KeyError): pass # No data in output.xml for this test @@ -1800,7 +1016,11 @@ def table_perf_trending_dash(table, input_data): if len(data_t) < 2: continue - classification_lst, avgs = classify_anomalies(data_t) + try: + classification_lst, avgs, _ = classify_anomalies(data_t) + except ValueError as err: + logging.info(f"{err} Skipping") + return win_size = min(len(data_t), table[u"window"]) long_win_size = min(len(data_t), table[u"long-trend-window"]) @@ -1814,6 +1034,13 @@ def table_perf_trending_dash(table, input_data): last_avg = avgs[-1] avg_week_ago = avgs[max(-win_size, -len(avgs))] + nr_of_last_avgs = 0; + for x in reversed(avgs): + if x == last_avg: + nr_of_last_avgs += 1 + else: + break + if isnan(last_avg) or isnan(avg_week_ago) or avg_week_ago == 0.0: rel_change_last = nan else: @@ -1835,27 +1062,23 @@ def table_perf_trending_dash(table, input_data): tbl_lst.append( [tbl_dict[tst_name][u"name"], round(last_avg / 1e6, 2), - rel_change_last, + nr_of_last_avgs, rel_change_long, - classification_lst[-win_size:].count(u"regression"), - classification_lst[-win_size:].count(u"progression")]) + classification_lst[-win_size+1:].count(u"regression"), + classification_lst[-win_size+1:].count(u"progression")]) tbl_lst.sort(key=lambda rel: rel[0]) - - tbl_sorted = list() - for nrr in range(table[u"window"], -1, -1): - tbl_reg = [item for item in tbl_lst if item[4] == nrr] - for nrp in range(table[u"window"], -1, -1): - tbl_out = [item for item in tbl_reg if item[5] == nrp] - tbl_out.sort(key=lambda rel: rel[2]) - tbl_sorted.extend(tbl_out) + tbl_lst.sort(key=lambda rel: rel[2]) + tbl_lst.sort(key=lambda rel: rel[3]) + tbl_lst.sort(key=lambda rel: rel[5], reverse=True) + tbl_lst.sort(key=lambda rel: rel[4], reverse=True) file_name = f"{table[u'output-file']}{table[u'output-file-ext']}" logging.info(f" Writing file: {file_name}") with open(file_name, u"wt") as file_handler: file_handler.write(header_str) - for test in tbl_sorted: + for test in tbl_lst: file_handler.write(u",".join([str(item) for item in test]) + u'\n') logging.info(f" Writing file: {table[u'output-file']}.txt") @@ -1890,6 +1113,8 @@ def _generate_url(testbed, test_name): nic = u"x553" elif u"cx556" in test_name or u"cx556a" in test_name: nic = u"cx556a" + elif u"ena" in test_name: + nic = u"nitro50g" else: nic = u"" @@ -1910,27 +1135,30 @@ def _generate_url(testbed, test_name): if u"1t1c" in test_name or \ (u"-1c-" in test_name and - testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")): + testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv", u"2n-tx2")): cores = u"1t1c" elif u"2t2c" in test_name or \ (u"-2c-" in test_name and - testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")): + testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv", u"2n-tx2")): cores = u"2t2c" elif u"4t4c" in test_name or \ (u"-4c-" in test_name and - testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv")): + testbed in (u"3n-hsw", u"3n-tsh", u"2n-dnv", u"3n-dnv", u"2n-tx2")): cores = u"4t4c" elif u"2t1c" in test_name or \ (u"-1c-" in test_name and - testbed in (u"2n-skx", u"3n-skx", u"2n-clx")): + testbed in + (u"2n-skx", u"3n-skx", u"2n-clx", u"2n-zn2", u"2n-aws", u"3n-aws")): cores = u"2t1c" elif u"4t2c" in test_name or \ (u"-2c-" in test_name and - testbed in (u"2n-skx", u"3n-skx", u"2n-clx")): + testbed in + (u"2n-skx", u"3n-skx", u"2n-clx", u"2n-zn2", u"2n-aws", u"3n-aws")): cores = u"4t2c" elif u"8t4c" in test_name or \ (u"-4c-" in test_name and - testbed in (u"2n-skx", u"3n-skx", u"2n-clx")): + testbed in + (u"2n-skx", u"3n-skx", u"2n-clx", u"2n-zn2", u"2n-aws", u"3n-aws")): cores = u"8t4c" else: cores = u"" @@ -1941,18 +1169,64 @@ def _generate_url(testbed, test_name): driver = u"l3fwd" elif u"avf" in test_name: driver = u"avf" + elif u"af-xdp" in test_name or u"af_xdp" in test_name: + driver = u"af_xdp" elif u"rdma" in test_name: driver = u"rdma" elif u"dnv" in testbed or u"tsh" in testbed: driver = u"ixgbe" + elif u"ena" in test_name: + driver = u"ena" else: driver = u"dpdk" - if u"acl" in test_name or \ - u"macip" in test_name or \ - u"nat" in test_name or \ - u"policer" in test_name or \ - u"cop" in test_name: + if u"macip-iacl1s" in test_name: + bsf = u"features-macip-iacl1" + elif u"macip-iacl10s" in test_name: + bsf = u"features-macip-iacl10" + elif u"macip-iacl50s" in test_name: + bsf = u"features-macip-iacl50" + elif u"iacl1s" in test_name: + bsf = u"features-iacl1" + elif u"iacl10s" in test_name: + bsf = u"features-iacl10" + elif u"iacl50s" in test_name: + bsf = u"features-iacl50" + elif u"oacl1s" in test_name: + bsf = u"features-oacl1" + elif u"oacl10s" in test_name: + bsf = u"features-oacl10" + elif u"oacl50s" in test_name: + bsf = u"features-oacl50" + elif u"nat44det" in test_name: + bsf = u"nat44det-bidir" + elif u"nat44ed" in test_name and u"udir" in test_name: + bsf = u"nat44ed-udir" + elif u"-cps" in test_name and u"ethip4udp" in test_name: + bsf = u"udp-cps" + elif u"-cps" in test_name and u"ethip4tcp" in test_name: + bsf = u"tcp-cps" + elif u"-pps" in test_name and u"ethip4udp" in test_name: + bsf = u"udp-pps" + elif u"-pps" in test_name and u"ethip4tcp" in test_name: + bsf = u"tcp-pps" + elif u"-tput" in test_name and u"ethip4udp" in test_name: + bsf = u"udp-tput" + elif u"-tput" in test_name and u"ethip4tcp" in test_name: + bsf = u"tcp-tput" + elif u"udpsrcscale" in test_name: + bsf = u"features-udp" + elif u"iacl" in test_name: + bsf = u"features" + elif u"policer" in test_name: + bsf = u"features" + elif u"adl" in test_name: + bsf = u"features" + elif u"cop" in test_name: + bsf = u"features" + elif u"nat" in test_name: + bsf = u"features" + elif u"macip" in test_name: bsf = u"features" elif u"scale" in test_name: bsf = u"scale" @@ -1963,6 +1237,24 @@ def _generate_url(testbed, test_name): if u"114b" in test_name and u"vhost" in test_name: domain = u"vts" + elif u"nat44" in test_name or u"-pps" in test_name or u"-cps" in test_name: + domain = u"nat44" + if u"nat44det" in test_name: + domain += u"-det-bidir" + else: + domain += u"-ed" + if u"udir" in test_name: + domain += u"-unidir" + elif u"-ethip4udp-" in test_name: + domain += u"-udp" + elif u"-ethip4tcp-" in test_name: + domain += u"-tcp" + if u"-cps" in test_name: + domain += u"-cps" + elif u"-pps" in test_name: + domain += u"-pps" + elif u"-tput" in test_name: + domain += u"-tput" elif u"testpmd" in test_name or u"l3fwd" in test_name: domain = u"dpdk" elif u"memif" in test_name: @@ -1989,8 +1281,12 @@ def _generate_url(testbed, test_name): bsf += u"-sw" elif u"hw" in test_name: bsf += u"-hw" + elif u"spe" in test_name: + bsf += u"-spe" elif u"ethip4vxlan" in test_name: domain = u"ip4_tunnels" + elif u"ethip4udpgeneve" in test_name: + domain = u"ip4_tunnels" elif u"ip4base" in test_name or u"ip4scale" in test_name: domain = u"ip4" elif u"ip6base" in test_name or u"ip6scale" in test_name: @@ -2026,15 +1322,33 @@ def table_perf_trending_dash_html(table, input_data): if not table.get(u"testbed", None): logging.error( f"The testbed is not defined for the table " - f"{table.get(u'title', u'')}." + f"{table.get(u'title', u'')}. Skipping." + ) + return + + test_type = table.get(u"test-type", u"MRR") + if test_type not in (u"MRR", u"NDR", u"PDR"): + logging.error( + f"Test type {table.get(u'test-type', u'MRR')} is not defined. " + f"Skipping." ) return + if test_type in (u"NDR", u"PDR"): + lnk_dir = u"../ndrpdr_trending/" + lnk_sufix = f"-{test_type.lower()}" + else: + lnk_dir = u"../trending/" + lnk_sufix = u"" + logging.info(f" Generating the table {table.get(u'title', u'')} ...") try: with open(table[u"input-file"], u'rt') as csv_file: csv_lst = list(csv.reader(csv_file, delimiter=u',', quotechar=u'"')) + except FileNotFoundError as err: + logging.warning(f"{err}") + return except KeyError: logging.warning(u"The input file is not defined.") return @@ -2089,13 +1403,14 @@ def table_perf_trending_dash_html(table, input_data): attrib=dict(align=u"left" if c_idx == 0 else u"center") ) # Name: - if c_idx == 0: + if c_idx == 0 and table.get(u"add-links", True): ref = ET.SubElement( tdata, u"a", attrib=dict( - href=f"../trending/" - f"{_generate_url(table.get(u'testbed', ''), item)}" + href=f"{lnk_dir}" + f"{_generate_url(table.get(u'testbed', ''), item)}" + f"{lnk_sufix}" ) ) ref.text = item @@ -2145,6 +1460,8 @@ def table_last_failed_tests(table, input_data): build = str(build) try: version = input_data.metadata(job, build).get(u"version", u"") + duration = \ + input_data.metadata(job, build).get(u"elapsedtime", u"") except KeyError: logging.error(f"Data for {job}: {build} is not present.") return @@ -2162,16 +1479,21 @@ def table_last_failed_tests(table, input_data): if not groups: continue nic = groups.group(0) - failed_tests.append(f"{nic}-{tst_data[u'name']}") - tbl_list.append(str(passed)) - tbl_list.append(str(failed)) + msg = tst_data[u'msg'].replace(u"\n", u"") + msg = re.sub(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})', + 'xxx.xxx.xxx.xxx', msg) + msg = msg.split(u'Also teardown failed')[0] + failed_tests.append(f"{nic}-{tst_data[u'name']}###{msg}") + tbl_list.append(passed) + tbl_list.append(failed) + tbl_list.append(duration) tbl_list.extend(failed_tests) file_name = f"{table[u'output-file']}{table[u'output-file-ext']}" logging.info(f" Writing file: {file_name}") with open(file_name, u"wt") as file_handler: for test in tbl_list: - file_handler.write(test + u'\n') + file_handler.write(f"{test}\n") def table_failed_tests(table, input_data): @@ -2193,6 +1515,10 @@ def table_failed_tests(table, input_data): ) data = input_data.filter_data(table, continue_on_error=True) + test_type = u"MRR" + if u"NDRPDR" in table.get(u"filter", list()): + test_type = u"NDRPDR" + # Prepare the header of the tables header = [ u"Test Case", @@ -2256,15 +1582,14 @@ def table_failed_tests(table, input_data): fails_last_csit = val[3] if fails_nr: max_fails = fails_nr if fails_nr > max_fails else max_fails - tbl_lst.append( - [ - tst_data[u"name"], - fails_nr, - fails_last_date, - fails_last_vpp, - f"mrr-daily-build-{fails_last_csit}" - ] - ) + tbl_lst.append([ + tst_data[u"name"], + fails_nr, + fails_last_date, + fails_last_vpp, + f"{u'mrr-daily' if test_type == u'MRR' else u'ndrpdr-weekly'}" + f"-build-{fails_last_csit}" + ]) tbl_lst.sort(key=lambda rel: rel[2], reverse=True) tbl_sorted = list() @@ -2298,10 +1623,25 @@ def table_failed_tests_html(table, input_data): if not table.get(u"testbed", None): logging.error( f"The testbed is not defined for the table " - f"{table.get(u'title', u'')}." + f"{table.get(u'title', u'')}. Skipping." + ) + return + + test_type = table.get(u"test-type", u"MRR") + if test_type not in (u"MRR", u"NDR", u"PDR", u"NDRPDR"): + logging.error( + f"Test type {table.get(u'test-type', u'MRR')} is not defined. " + f"Skipping." ) return + if test_type in (u"NDRPDR", u"NDR", u"PDR"): + lnk_dir = u"../ndrpdr_trending/" + lnk_sufix = u"-pdr" + else: + lnk_dir = u"../trending/" + lnk_sufix = u"" + logging.info(f" Generating the table {table.get(u'title', u'')} ...") try: @@ -2343,13 +1683,14 @@ def table_failed_tests_html(table, input_data): attrib=dict(align=u"left" if c_idx == 0 else u"center") ) # Name: - if c_idx == 0: + if c_idx == 0 and table.get(u"add-links", True): ref = ET.SubElement( tdata, u"a", attrib=dict( - href=f"../trending/" - f"{_generate_url(table.get(u'testbed', ''), item)}" + href=f"{lnk_dir}" + f"{_generate_url(table.get(u'testbed', ''), item)}" + f"{lnk_sufix}" ) ) ref.text = item @@ -2395,9 +1736,17 @@ def table_comparison(table, input_data): if col.get(u"data-set", None) is None: logging.warning(f"No data for column {col.get(u'title', u'')}") continue + tag = col.get(u"tag", None) data = input_data.filter_data( table, - params=[u"throughput", u"result", u"name", u"parent", u"tags"], + params=[ + u"throughput", + u"result", + u"latency", + u"name", + u"parent", + u"tags" + ], data=col[u"data-set"], continue_on_error=True ) @@ -2408,8 +1757,11 @@ def table_comparison(table, input_data): for builds in data.values: for build in builds: for tst_name, tst_data in build.items(): + if tag and tag not in tst_data[u"tags"]: + continue tst_name_mod = \ - _tpc_modify_test_name(tst_name).replace(u"2n1l-", u"") + _tpc_modify_test_name(tst_name, ignore_nic=True).\ + replace(u"2n1l-", u"") if col_data[u"data"].get(tst_name_mod, None) is None: name = tst_data[u'name'].rsplit(u'-', 1)[0] if u"across testbeds" in table[u"title"].lower() or \ @@ -2423,7 +1775,7 @@ def table_comparison(table, input_data): u"stdev": None } _tpc_insert_data( - target=col_data[u"data"][tst_name_mod][u"data"], + target=col_data[u"data"][tst_name_mod], src=tst_data, include_tests=table[u"include-tests"] ) @@ -2432,15 +1784,24 @@ def table_comparison(table, input_data): if replacement: rpl_data = input_data.filter_data( table, - params=[u"throughput", u"result", u"name", u"parent", u"tags"], + params=[ + u"throughput", + u"result", + u"latency", + u"name", + u"parent", + u"tags" + ], data=replacement, continue_on_error=True ) for builds in rpl_data.values: for build in builds: for tst_name, tst_data in build.items(): + if tag and tag not in tst_data[u"tags"]: + continue tst_name_mod = \ - _tpc_modify_test_name(tst_name).\ + _tpc_modify_test_name(tst_name, ignore_nic=True).\ replace(u"2n1l-", u"") if col_data[u"data"].get(tst_name_mod, None) is None: name = tst_data[u'name'].rsplit(u'-', 1)[0] @@ -2459,21 +1820,17 @@ def table_comparison(table, input_data): col_data[u"data"][tst_name_mod][u"replace"] = False col_data[u"data"][tst_name_mod][u"data"] = list() _tpc_insert_data( - target=col_data[u"data"][tst_name_mod][u"data"], + target=col_data[u"data"][tst_name_mod], src=tst_data, include_tests=table[u"include-tests"] ) - if table[u"include-tests"] in (u"NDR", u"PDR"): + if table[u"include-tests"] in (u"NDR", u"PDR") or \ + u"latency" in table[u"include-tests"]: for tst_name, tst_data in col_data[u"data"].items(): if tst_data[u"data"]: tst_data[u"mean"] = mean(tst_data[u"data"]) tst_data[u"stdev"] = stdev(tst_data[u"data"]) - elif table[u"include-tests"] in (u"MRR", ): - for tst_name, tst_data in col_data[u"data"].items(): - if tst_data[u"data"]: - tst_data[u"mean"] = tst_data[u"data"][0] - tst_data[u"stdev"] = tst_data[u"data"][0] cols.append(col_data) @@ -2489,6 +1846,10 @@ def table_comparison(table, input_data): u"stdev": tst_data[u"stdev"] } + if not tbl_dict: + logging.warning(f"No data for table {table.get(u'title', u'')}!") + return + tbl_lst = list() for tst_data in tbl_dict.values(): row = [tst_data[u"name"], ] @@ -2497,6 +1858,7 @@ def table_comparison(table, input_data): tbl_lst.append(row) comparisons = table.get(u"comparisons", None) + rcas = list() if comparisons and isinstance(comparisons, list): for idx, comp in enumerate(comparisons): try: @@ -2506,19 +1868,38 @@ def table_comparison(table, input_data): logging.warning(u"Comparison: No references defined! Skipping.") comparisons.pop(idx) continue - if not (0 < col_ref <= len(cols) and - 0 < col_cmp <= len(cols)) or \ - col_ref == col_cmp: + if not (0 < col_ref <= len(cols) and 0 < col_cmp <= len(cols) or + col_ref == col_cmp): logging.warning(f"Wrong values of reference={col_ref} " f"and/or compare={col_cmp}. Skipping.") comparisons.pop(idx) continue + rca_file_name = comp.get(u"rca-file", None) + if rca_file_name: + try: + with open(rca_file_name, u"r") as file_handler: + rcas.append( + { + u"title": f"RCA{idx + 1}", + u"data": load(file_handler, Loader=FullLoader) + } + ) + except (YAMLError, IOError) as err: + logging.warning( + f"The RCA file {rca_file_name} does not exist or " + f"it is corrupted!" + ) + logging.debug(repr(err)) + rcas.append(None) + else: + rcas.append(None) + else: + comparisons = None tbl_cmp_lst = list() if comparisons: for row in tbl_lst: new_row = deepcopy(row) - add_to_tbl = False for comp in comparisons: ref_itm = row[int(comp[u"reference"])] if ref_itm is None and \ @@ -2530,55 +1911,45 @@ def table_comparison(table, input_data): cmp_itm[u"mean"] is not None and \ ref_itm[u"stdev"] is not None and \ cmp_itm[u"stdev"] is not None: - delta, d_stdev = relative_change_stdev( - ref_itm[u"mean"], cmp_itm[u"mean"], - ref_itm[u"stdev"], cmp_itm[u"stdev"] - ) - new_row.append( - { - u"mean": delta * 1e6, - u"stdev": d_stdev * 1e6 - } - ) - add_to_tbl = True + try: + delta, d_stdev = relative_change_stdev( + ref_itm[u"mean"], cmp_itm[u"mean"], + ref_itm[u"stdev"], cmp_itm[u"stdev"] + ) + except ZeroDivisionError: + break + if delta is None or math.isnan(delta): + break + new_row.append({ + u"mean": delta * 1e6, + u"stdev": d_stdev * 1e6 + }) else: - new_row.append(None) - if add_to_tbl: + break + else: tbl_cmp_lst.append(new_row) - tbl_cmp_lst.sort(key=lambda rel: rel[0], reverse=False) - tbl_cmp_lst.sort(key=lambda rel: rel[-1][u'mean'], reverse=True) - - rcas = list() - rca_in = table.get(u"rca", None) - if rca_in and isinstance(rca_in, list): - for idx, itm in enumerate(rca_in): - try: - with open(itm.get(u"data", u""), u"r") as rca_file: - rcas.append( - { - u"title": itm.get(u"title", f"RCA{idx}"), - u"data": load(rca_file, Loader=FullLoader) - } - ) - except (YAMLError, IOError) as err: - logging.warning( - f"The RCA file {itm.get(u'data', u'')} does not exist or " - f"it is corrupted!" - ) - logging.debug(repr(err)) + try: + tbl_cmp_lst.sort(key=lambda rel: rel[0], reverse=False) + tbl_cmp_lst.sort(key=lambda rel: rel[-1][u'mean'], reverse=True) + except TypeError as err: + logging.warning(f"Empty data element in table\n{tbl_cmp_lst}\n{err}") tbl_for_csv = list() for line in tbl_cmp_lst: row = [line[0], ] for idx, itm in enumerate(line[1:]): - if itm is None: + if itm is None or not isinstance(itm, dict) or\ + itm.get(u'mean', None) is None or \ + itm.get(u'stdev', None) is None: row.append(u"NT") row.append(u"NT") else: row.append(round(float(itm[u'mean']) / 1e6, 3)) row.append(round(float(itm[u'stdev']) / 1e6, 3)) for rca in rcas: + if rca is None: + continue rca_nr = rca[u"data"].get(row[0], u"-") row.append(f"[{rca_nr}]" if rca_nr != u"-" else u"-") tbl_for_csv.append(row) @@ -2589,12 +1960,14 @@ def table_comparison(table, input_data): header_csv.append(f"Stdev({col[u'title']})") for comp in comparisons: header_csv.append( - f"Avg({comp.get(u'title', u'')}" + f"Avg({comp.get(u'title', u'')})" ) header_csv.append( f"Stdev({comp.get(u'title', u'')})" ) - header_csv.extend([rca[u"title"] for rca in rcas]) + for rca in rcas: + if rca: + header_csv.append(rca[u"title"]) legend_lst = table.get(u"legend", None) if legend_lst is None: @@ -2603,12 +1976,14 @@ def table_comparison(table, input_data): legend = u"\n" + u"\n".join(legend_lst) + u"\n" footnote = u"" - for rca in rcas: - footnote += f"\n{rca[u'title']}:\n" - footnote += rca[u"data"].get(u"footnote", u"") + if rcas and any(rcas): + footnote += u"\nRoot Cause Analysis:\n" + for rca in rcas: + if rca: + footnote += f"{rca[u'data'].get(u'footnote', u'')}\n" - csv_file = f"{table[u'output-file']}-csv.csv" - with open(csv_file, u"wt", encoding='utf-8') as file_handler: + csv_file_name = f"{table[u'output-file']}-csv.csv" + with open(csv_file_name, u"wt", encoding='utf-8') as file_handler: file_handler.write( u",".join([f'"{itm}"' for itm in header_csv]) + u"\n" ) @@ -2628,19 +2003,21 @@ def table_comparison(table, input_data): for line in tbl_cmp_lst: row = [line[0], ] for idx, itm in enumerate(line[1:]): - if itm is None: + if itm is None or not isinstance(itm, dict) or \ + itm.get(u'mean', None) is None or \ + itm.get(u'stdev', None) is None: new_itm = u"NT" else: if idx < len(cols): new_itm = ( - f"{round(float(itm[u'mean']) / 1e6, 1)} " - f"\u00B1{round(float(itm[u'stdev']) / 1e6, 1)}". + f"{round(float(itm[u'mean']) / 1e6, 2)} " + f"\u00B1{round(float(itm[u'stdev']) / 1e6, 2)}". replace(u"nan", u"NaN") ) else: new_itm = ( - f"{round(float(itm[u'mean']) / 1e6, 1):+} " - f"\u00B1{round(float(itm[u'stdev']) / 1e6, 1)}". + f"{round(float(itm[u'mean']) / 1e6, 2):+} " + f"\u00B1{round(float(itm[u'stdev']) / 1e6, 2)}". replace(u"nan", u"NaN") ) if len(new_itm.rsplit(u" ", 1)[-1]) > max_lens[idx]: @@ -2649,6 +2026,10 @@ def table_comparison(table, input_data): tbl_tmp.append(row) + header = [u"Test Case", ] + header.extend([col[u"title"] for col in cols]) + header.extend([comp.get(u"title", u"") for comp in comparisons]) + tbl_final = list() for line in tbl_tmp: row = [line[0], ] @@ -2659,34 +2040,43 @@ def table_comparison(table, input_data): itm_lst = itm.rsplit(u"\u00B1", 1) itm_lst[-1] = \ f"{u' ' * (max_lens[idx] - len(itm_lst[-1]))}{itm_lst[-1]}" - row.append(u"\u00B1".join(itm_lst)) - for rca in rcas: - rca_nr = rca[u"data"].get(row[0], u"-") - row.append(f"[{rca_nr}]" if rca_nr != u"-" else u"-") - + itm_str = u"\u00B1".join(itm_lst) + + if idx >= len(cols): + # Diffs + rca = rcas[idx - len(cols)] + if rca: + # Add rcas to diffs + rca_nr = rca[u"data"].get(row[0], None) + if rca_nr: + hdr_len = len(header[idx + 1]) - 1 + if hdr_len < 19: + hdr_len = 19 + rca_nr = f"[{rca_nr}]" + itm_str = ( + f"{u' ' * (4 - len(rca_nr))}{rca_nr}" + f"{u' ' * (hdr_len - 4 - len(itm_str))}" + f"{itm_str}" + ) + row.append(itm_str) tbl_final.append(row) - header = [u"Test Case", ] - header.extend([col[u"title"] for col in cols]) - header.extend([comp.get(u"title", u"") for comp in comparisons]) - header.extend([rca[u"title"] for rca in rcas]) - # Generate csv tables: - csv_file = f"{table[u'output-file']}.csv" - with open(csv_file, u"wt", encoding='utf-8') as file_handler: + csv_file_name = f"{table[u'output-file']}.csv" + logging.info(f" Writing the file {csv_file_name}") + with open(csv_file_name, u"wt", encoding='utf-8') as file_handler: file_handler.write(u";".join(header) + u"\n") for test in tbl_final: file_handler.write(u";".join([str(item) for item in test]) + u"\n") # Generate txt table: txt_file_name = f"{table[u'output-file']}.txt" - convert_csv_to_pretty_txt(csv_file, txt_file_name, delimiter=u";") + logging.info(f" Writing the file {txt_file_name}") + convert_csv_to_pretty_txt(csv_file_name, txt_file_name, delimiter=u";") - with open(txt_file_name, u'a', encoding='utf-8') as txt_file: - txt_file.write(legend) - if footnote: - txt_file.write(footnote) - txt_file.write(u":END") + with open(txt_file_name, u'a', encoding='utf-8') as file_handler: + file_handler.write(legend) + file_handler.write(footnote) # Generate html table: _tpc_generate_html_table( @@ -2698,3 +2088,181 @@ def table_comparison(table, input_data): sort_data=False, title=table.get(u"title", u"") ) + + +def table_weekly_comparison(table, in_data): + """Generate the table(s) with algorithm: table_weekly_comparison + specified in the specification file. + + :param table: Table to generate. + :param in_data: Data to process. + :type table: pandas.Series + :type in_data: InputData + """ + logging.info(f" Generating the table {table.get(u'title', u'')} ...") + + # Transform the data + logging.info( + f" Creating the data set for the {table.get(u'type', u'')} " + f"{table.get(u'title', u'')}." + ) + + incl_tests = table.get(u"include-tests", None) + if incl_tests not in (u"NDR", u"PDR"): + logging.error(f"Wrong tests to include specified ({incl_tests}).") + return + + nr_cols = table.get(u"nr-of-data-columns", None) + if not nr_cols or nr_cols < 2: + logging.error( + f"No columns specified for {table.get(u'title', u'')}. Skipping." + ) + return + + data = in_data.filter_data( + table, + params=[u"throughput", u"result", u"name", u"parent", u"tags"], + continue_on_error=True + ) + + header = [ + [u"VPP Version", ], + [u"Start Timestamp", ], + [u"CSIT Build", ], + [u"CSIT Testbed", ] + ] + tbl_dict = dict() + idx = 0 + tb_tbl = table.get(u"testbeds", None) + for job_name, job_data in data.items(): + for build_nr, build in job_data.items(): + if idx >= nr_cols: + break + if build.empty: + continue + + tb_ip = in_data.metadata(job_name, build_nr).get(u"testbed", u"") + if tb_ip and tb_tbl: + testbed = tb_tbl.get(tb_ip, u"") + else: + testbed = u"" + header[2].insert(1, build_nr) + header[3].insert(1, testbed) + header[1].insert( + 1, in_data.metadata(job_name, build_nr).get(u"generated", u"") + ) + header[0].insert( + 1, in_data.metadata(job_name, build_nr).get(u"version", u"") + ) + + for tst_name, tst_data in build.items(): + tst_name_mod = \ + _tpc_modify_test_name(tst_name).replace(u"2n1l-", u"") + if not tbl_dict.get(tst_name_mod, None): + tbl_dict[tst_name_mod] = dict( + name=tst_data[u'name'].rsplit(u'-', 1)[0], + ) + try: + tbl_dict[tst_name_mod][-idx - 1] = \ + tst_data[u"throughput"][incl_tests][u"LOWER"] + except (TypeError, IndexError, KeyError, ValueError): + pass + idx += 1 + + if idx < nr_cols: + logging.error(u"Not enough data to build the table! Skipping") + return + + cmp_dict = dict() + for idx, cmp in enumerate(table.get(u"comparisons", list())): + idx_ref = cmp.get(u"reference", None) + idx_cmp = cmp.get(u"compare", None) + if idx_ref is None or idx_cmp is None: + continue + header[0].append( + f"Diff({header[0][idx_ref - idx].split(u'~')[-1]} vs " + f"{header[0][idx_cmp - idx].split(u'~')[-1]})" + ) + header[1].append(u"") + header[2].append(u"") + header[3].append(u"") + for tst_name, tst_data in tbl_dict.items(): + if not cmp_dict.get(tst_name, None): + cmp_dict[tst_name] = list() + ref_data = tst_data.get(idx_ref, None) + cmp_data = tst_data.get(idx_cmp, None) + if ref_data is None or cmp_data is None: + cmp_dict[tst_name].append(float(u'nan')) + else: + cmp_dict[tst_name].append( + relative_change(ref_data, cmp_data) + ) + + tbl_lst_none = list() + tbl_lst = list() + for tst_name, tst_data in tbl_dict.items(): + itm_lst = [tst_data[u"name"], ] + for idx in range(nr_cols): + item = tst_data.get(-idx - 1, None) + if item is None: + itm_lst.insert(1, None) + else: + itm_lst.insert(1, round(item / 1e6, 1)) + itm_lst.extend( + [ + None if itm is None else round(itm, 1) + for itm in cmp_dict[tst_name] + ] + ) + if str(itm_lst[-1]) == u"nan" or itm_lst[-1] is None: + tbl_lst_none.append(itm_lst) + else: + tbl_lst.append(itm_lst) + + tbl_lst_none.sort(key=lambda rel: rel[0], reverse=False) + tbl_lst.sort(key=lambda rel: rel[0], reverse=False) + tbl_lst.sort(key=lambda rel: rel[-1], reverse=False) + tbl_lst.extend(tbl_lst_none) + + # Generate csv table: + csv_file_name = f"{table[u'output-file']}.csv" + logging.info(f" Writing the file {csv_file_name}") + with open(csv_file_name, u"wt", encoding='utf-8') as file_handler: + for hdr in header: + file_handler.write(u",".join(hdr) + u"\n") + for test in tbl_lst: + file_handler.write(u",".join( + [ + str(item).replace(u"None", u"-").replace(u"nan", u"-"). + replace(u"null", u"-") for item in test + ] + ) + u"\n") + + txt_file_name = f"{table[u'output-file']}.txt" + logging.info(f" Writing the file {txt_file_name}") + convert_csv_to_pretty_txt(csv_file_name, txt_file_name, delimiter=u",") + + # Reorganize header in txt table + txt_table = list() + with open(txt_file_name, u"rt", encoding='utf-8') as file_handler: + for line in list(file_handler): + txt_table.append(line) + try: + txt_table.insert(5, txt_table.pop(2)) + with open(txt_file_name, u"wt", encoding='utf-8') as file_handler: + file_handler.writelines(txt_table) + except IndexError: + pass + + # Generate html table: + hdr_html = [ + u"