X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=blobdiff_plain;f=resources%2Ftools%2Fpresentation%2Fgenerator_tables.py;h=13c8efffdb480e64e295141d3960fd9c0364c1f3;hp=9f0096557e6892a6b675f2c5909c4f96c09507fa;hb=9821b058c2f4901a9b4d66667018da214513ab28;hpb=295a68551716af397597bc721d6a3115572009f9 diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py index 9f0096557e..13c8efffdb 100644 --- a/resources/tools/presentation/generator_tables.py +++ b/resources/tools/presentation/generator_tables.py @@ -22,6 +22,7 @@ import pandas as pd from string import replace from math import isnan +from xml.etree import ElementTree as ET from errors import PresentationError from utils import mean, stdev, relative_change, remove_outliers, find_outliers @@ -548,9 +549,9 @@ def table_performance_trending_dashboard(table, input_data): # Prepare the header of the tables header = ["Test case", "Thput trend [Mpps]", - "Change [Mpps]", + "Anomaly [Mpps]", "Change [%]", - "Anomaly"] + "Classification"] header_str = ",".join(header) + "\n" # Prepare data to the table: @@ -567,7 +568,7 @@ def table_performance_trending_dashboard(table, input_data): try: tbl_dict[tst_name]["data"]. \ append(tst_data["result"]["throughput"]) - except TypeError: + except (TypeError, KeyError): pass # No data in output.xml for this test tbl_lst = list() @@ -579,40 +580,46 @@ def table_performance_trending_dashboard(table, input_data): # Test name: name = tbl_dict[tst_name]["name"] # Throughput trend: - trend = list(pd_data.rolling(window=win_size).median())[-2] + trend = list(pd_data.rolling(window=win_size, min_periods=2). + median())[-2] # Anomaly: t_data, _ = find_outliers(pd_data) last = list(t_data)[-1] t_stdev = list(t_data.rolling(window=win_size, min_periods=2). std())[-2] if isnan(last): - anomaly = "outlier" + classification = "outlier" + last = list(pd_data)[-1] elif last < (trend - 3 * t_stdev): - anomaly = "regression" + classification = "regression" elif last > (trend + 3 * t_stdev): - anomaly = "progression" + classification = "progression" else: - anomaly = "normal" - # Change: - change = round(float(last - trend) / 1000000, 2) - # Relative change: - rel_change = int(relative_change(float(trend), float(last))) - - tbl_lst.append([name, - round(float(last) / 1000000, 2), - change, - rel_change, - anomaly]) + classification = "normal" - # Sort the table according to the relative change - tbl_lst.sort(key=lambda rel: rel[-2], reverse=True) + if not isnan(last) and not isnan(trend) and trend != 0: + # Relative change: + rel_change = int(relative_change(float(trend), float(last))) + + tbl_lst.append([name, + round(float(trend) / 1000000, 2), + round(float(last) / 1000000, 2), + rel_change, + classification]) + + # Sort the table according to the classification + tbl_sorted = list() + for classification in ("regression", "outlier", "progression", "normal"): + tbl_tmp = [item for item in tbl_lst if item[4] == classification] + tbl_tmp.sort(key=lambda rel: rel[0]) + tbl_sorted.extend(tbl_tmp) - file_name = "{0}.{1}".format(table["output-file"], table["output-file-ext"]) + file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"]) logging.info(" Writing file: '{0}'".format(file_name)) with open(file_name, "w") as file_handler: file_handler.write(header_str) - for test in tbl_lst: + for test in tbl_sorted: file_handler.write(",".join([str(item) for item in test]) + '\n') txt_file_name = "{0}.txt".format(table["output-file"]) @@ -628,3 +635,69 @@ def table_performance_trending_dashboard(table, input_data): txt_table.align["Test case"] = "l" with open(txt_file_name, "w") as txt_file: txt_file.write(str(txt_table)) + + +def table_performance_trending_dashboard_html(table, input_data): + """Generate the table(s) with algorithm: + table_performance_trending_dashboard_html specified in the specification + file. + + :param table: Table to generate. + :param input_data: Data to process. + :type table: pandas.Series + :type input_data: InputData + """ + + logging.info(" Generating the table {0} ...". + format(table.get("title", ""))) + + try: + with open(table["input-file"], 'rb') as csv_file: + csv_content = csv.reader(csv_file, delimiter=',', quotechar='"') + csv_lst = [item for item in csv_content] + except KeyError: + logging.warning("The input file is not defined.") + return + except csv.Error as err: + logging.warning("Not possible to process the file '{0}'.\n{1}". + format(table["input-file"], err)) + return + + # Table: + dashboard = ET.Element("table", attrib=dict(width="100%", border='0')) + + # Table header: + tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor="#6699ff")) + for idx, item in enumerate(csv_lst[0]): + alignment = "left" if idx == 0 else "right" + th = ET.SubElement(tr, "th", attrib=dict(align=alignment)) + th.text = item + + # Rows: + for r_idx, row in enumerate(csv_lst[1:]): + background = "#D4E4F7" if r_idx % 2 else "white" + tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor=background)) + + # Columns: + for c_idx, item in enumerate(row): + alignment = "left" if c_idx == 0 else "center" + td = ET.SubElement(tr, "td", attrib=dict(align=alignment)) + if c_idx == 4: + if item == "regression": + td.set("bgcolor", "#FF0000") + elif item == "outlier": + td.set("bgcolor", "#818181") + elif item == "progression": + td.set("bgcolor", "#008000") + td.text = item + + try: + with open(table["output-file"], 'w') as html_file: + logging.info(" Writing file: '{0}'". + format(table["output-file"])) + html_file.write(".. raw:: html\n\n\t") + html_file.write(ET.tostring(dashboard)) + html_file.write("\n\t



\n") + except KeyError: + logging.warning("The output file is not defined.") + return