X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=resources%2Ftools%2Fpresentation%2Fgenerator_tables.py;h=05a696c2144246a1391ea751b9176dc309b25b59;hb=494c42cf77c2571a64854256a68abd7f180cd5d6;hp=12cbee2dae95eb1e417f6c7a942f64d27e640ca3;hpb=23731f392ad8705b17cf37f9c2d397b20305f924;p=csit.git diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py index 12cbee2dae..05a696c214 100644 --- a/resources/tools/presentation/generator_tables.py +++ b/resources/tools/presentation/generator_tables.py @@ -1,4 +1,4 @@ -# Copyright (c) 2017 Cisco and/or its affiliates. +# Copyright (c) 2019 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -17,15 +17,20 @@ import logging import csv -import prettytable -import pandas as pd +import re from string import replace -from math import isnan +from collections import OrderedDict +from numpy import nan, isnan from xml.etree import ElementTree as ET +from datetime import datetime as dt +from datetime import timedelta -from errors import PresentationError -from utils import mean, stdev, relative_change, remove_outliers, find_outliers +from utils import mean, stdev, relative_change, classify_anomalies, \ + convert_csv_to_pretty_txt, relative_change_stdev + + +REGEX_NIC = re.compile(r'\d*ge\dp\d\D*\d*') def generate_tables(spec, data): @@ -41,9 +46,9 @@ def generate_tables(spec, data): for table in spec.tables: try: eval(table["algorithm"])(table, data) - except NameError: - logging.error("The algorithm '{0}' is not defined.". - format(table["algorithm"])) + except NameError as err: + logging.error("Probably algorithm '{alg}' is not defined: {err}". + format(alg=table["algorithm"], err=repr(err))) logging.info("Done.") @@ -61,6 +66,8 @@ def table_details(table, input_data): format(table.get("title", ""))) # Transform the data + logging.info(" Creating the data set for the {0} '{1}'.". + format(table.get("type", ""), table.get("title", ""))) data = input_data.filter_data(table) # Prepare the header of the tables @@ -89,7 +96,7 @@ def table_details(table, input_data): try: col_data = str(data[job][build][test][column["data"]. split(" ")[1]]).replace('"', '""') - if column["data"].split(" ")[1] in ("vat-history", + if column["data"].split(" ")[1] in ("conf-history", "show-run"): col_data = replace(col_data, " |br| ", "", maxreplace=1) @@ -127,10 +134,14 @@ def table_merged_details(table, input_data): format(table.get("title", ""))) # Transform the data + logging.info(" Creating the data set for the {0} '{1}'.". + format(table.get("type", ""), table.get("title", ""))) data = input_data.filter_data(table) data = input_data.merge_data(data) data.sort_index(inplace=True) + logging.info(" Creating the data set for the {0} '{1}'.". + format(table.get("type", ""), table.get("title", ""))) suites = input_data.filter_data(table, data_set="suites") suites = input_data.merge_data(suites) @@ -150,7 +161,9 @@ def table_merged_details(table, input_data): try: col_data = str(data[test][column["data"]. split(" ")[1]]).replace('"', '""') - if column["data"].split(" ")[1] in ("vat-history", + col_data = replace(col_data, "No Data", + "Not Captured ") + if column["data"].split(" ")[1] in ("conf-history", "show-run"): col_data = replace(col_data, " |br| ", "", maxreplace=1) @@ -158,7 +171,7 @@ def table_merged_details(table, input_data): format(col_data[:-5]) row_lst.append('"{0}"'.format(col_data)) except KeyError: - row_lst.append("No data") + row_lst.append('"Not captured"') table_lst.append(row_lst) # Write the data to file @@ -174,8 +187,8 @@ def table_merged_details(table, input_data): logging.info(" Done.") -def table_performance_improvements(table, input_data): - """Generate the table(s) with algorithm: table_performance_improvements +def table_performance_comparison(table, input_data): + """Generate the table(s) with algorithm: table_performance_comparison specified in the specification file. :param table: Table to generate. @@ -184,165 +197,336 @@ def table_performance_improvements(table, input_data): :type input_data: InputData """ - def _write_line_to_file(file_handler, data): - """Write a line to the .csv file. - - :param file_handler: File handler for the csv file. It must be open for - writing text. - :param data: Item to be written to the file. - :type file_handler: BinaryIO - :type data: list - """ - - line_lst = list() - for item in data: - if isinstance(item["data"], str): - # Remove -?drdisc from the end - if item["data"].endswith("drdisc"): - item["data"] = item["data"][:-8] - line_lst.append(item["data"]) - elif isinstance(item["data"], float): - line_lst.append("{:.1f}".format(item["data"])) - elif item["data"] is None: - line_lst.append("") - file_handler.write(",".join(line_lst) + "\n") - logging.info(" Generating the table {0} ...". format(table.get("title", ""))) - # Read the template - file_name = table.get("template", None) - if file_name: - try: - tmpl = _read_csv_template(file_name) - except PresentationError: - logging.error(" The template '{0}' does not exist. Skipping the " - "table.".format(file_name)) - return None - else: - logging.error("The template is not defined. Skipping the table.") - return None - # Transform the data - data = input_data.filter_data(table) + logging.info(" Creating the data set for the {0} '{1}'.". + format(table.get("type", ""), table.get("title", ""))) + data = input_data.filter_data(table, continue_on_error=True) # Prepare the header of the tables - header = list() - for column in table["columns"]: - header.append(column["title"]) + try: + header = ["Test case", ] - # Generate the data for the table according to the model in the table - # specification - tbl_lst = list() - for tmpl_item in tmpl: - tbl_item = list() - for column in table["columns"]: - cmd = column["data"].split(" ")[0] - args = column["data"].split(" ")[1:] - if cmd == "template": + if table["include-tests"] == "MRR": + hdr_param = "Receive Rate" + else: + hdr_param = "Throughput" + + history = table.get("history", None) + if history: + for item in history: + header.extend( + ["{0} {1} [Mpps]".format(item["title"], hdr_param), + "{0} Stdev [Mpps]".format(item["title"])]) + header.extend( + ["{0} {1} [Mpps]".format(table["reference"]["title"], hdr_param), + "{0} Stdev [Mpps]".format(table["reference"]["title"]), + "{0} {1} [Mpps]".format(table["compare"]["title"], hdr_param), + "{0} Stdev [Mpps]".format(table["compare"]["title"]), + "Delta [%]"]) + header_str = ",".join(header) + "\n" + except (AttributeError, KeyError) as err: + logging.error("The model is invalid, missing parameter: {0}". + format(err)) + return + + # Prepare data to the table: + tbl_dict = dict() + for job, builds in table["reference"]["data"].items(): + for build in builds: + for tst_name, tst_data in data[job][str(build)].iteritems(): + tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\ + replace("-ndrpdr", "").replace("-pdrdisc", "").\ + replace("-ndrdisc", "").replace("-pdr", "").\ + replace("-ndr", "").\ + replace("1t1c", "1c").replace("2t1c", "1c").\ + replace("2t2c", "2c").replace("4t2c", "2c").\ + replace("4t4c", "4c").replace("8t4c", "4c") + if "across topologies" in table["title"].lower(): + tst_name_mod = tst_name_mod.replace("2n1l-", "") + if tbl_dict.get(tst_name_mod, None) is None: + groups = re.search(REGEX_NIC, tst_data["parent"]) + nic = groups.group(0) if groups else "" + name = "{0}-{1}".format(nic, "-".join(tst_data["name"]. + split("-")[:-1])) + if "across testbeds" in table["title"].lower() or \ + "across topologies" in table["title"].lower(): + name = name.\ + replace("1t1c", "1c").replace("2t1c", "1c").\ + replace("2t2c", "2c").replace("4t2c", "2c").\ + replace("4t4c", "4c").replace("8t4c", "4c") + tbl_dict[tst_name_mod] = {"name": name, + "ref-data": list(), + "cmp-data": list()} try: - val = float(tmpl_item[int(args[0])]) - except ValueError: - val = tmpl_item[int(args[0])] - tbl_item.append({"data": val}) - elif cmd == "data": - jobs = args[0:-1] - operation = args[-1] - data_lst = list() - for job in jobs: - for build in data[job]: - try: - data_lst.append(float(build[tmpl_item[0]] - ["throughput"]["value"])) - except (KeyError, TypeError): - # No data, ignore - continue - if data_lst: - tbl_item.append({"data": (eval(operation)(data_lst)) / - 1000000}) - else: - tbl_item.append({"data": None}) - elif cmd == "operation": - operation = args[0] + # TODO: Re-work when NDRPDRDISC tests are not used + if table["include-tests"] == "MRR": + tbl_dict[tst_name_mod]["ref-data"]. \ + append(tst_data["result"]["receive-rate"].avg) + elif table["include-tests"] == "PDR": + if tst_data["type"] == "PDR": + tbl_dict[tst_name_mod]["ref-data"]. \ + append(tst_data["throughput"]["value"]) + elif tst_data["type"] == "NDRPDR": + tbl_dict[tst_name_mod]["ref-data"].append( + tst_data["throughput"]["PDR"]["LOWER"]) + elif table["include-tests"] == "NDR": + if tst_data["type"] == "NDR": + tbl_dict[tst_name_mod]["ref-data"]. \ + append(tst_data["throughput"]["value"]) + elif tst_data["type"] == "NDRPDR": + tbl_dict[tst_name_mod]["ref-data"].append( + tst_data["throughput"]["NDR"]["LOWER"]) + else: + continue + except TypeError: + pass # No data in output.xml for this test + + for job, builds in table["compare"]["data"].items(): + for build in builds: + for tst_name, tst_data in data[job][str(build)].iteritems(): + tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \ + replace("-ndrpdr", "").replace("-pdrdisc", ""). \ + replace("-ndrdisc", "").replace("-pdr", ""). \ + replace("-ndr", "").\ + replace("1t1c", "1c").replace("2t1c", "1c").\ + replace("2t2c", "2c").replace("4t2c", "2c").\ + replace("4t4c", "4c").replace("8t4c", "4c") + if "across topologies" in table["title"].lower(): + tst_name_mod = tst_name_mod.replace("2n1l-", "") try: - nr1 = float(tbl_item[int(args[1])]["data"]) - nr2 = float(tbl_item[int(args[2])]["data"]) - if nr1 and nr2: - tbl_item.append({"data": eval(operation)(nr1, nr2)}) + # TODO: Re-work when NDRPDRDISC tests are not used + if table["include-tests"] == "MRR": + tbl_dict[tst_name_mod]["cmp-data"]. \ + append(tst_data["result"]["receive-rate"].avg) + elif table["include-tests"] == "PDR": + if tst_data["type"] == "PDR": + tbl_dict[tst_name_mod]["cmp-data"]. \ + append(tst_data["throughput"]["value"]) + elif tst_data["type"] == "NDRPDR": + tbl_dict[tst_name_mod]["cmp-data"].append( + tst_data["throughput"]["PDR"]["LOWER"]) + elif table["include-tests"] == "NDR": + if tst_data["type"] == "NDR": + tbl_dict[tst_name_mod]["cmp-data"]. \ + append(tst_data["throughput"]["value"]) + elif tst_data["type"] == "NDRPDR": + tbl_dict[tst_name_mod]["cmp-data"].append( + tst_data["throughput"]["NDR"]["LOWER"]) else: - tbl_item.append({"data": None}) - except (IndexError, ValueError, TypeError): - logging.error("No data for {0}".format(tbl_item[0]["data"])) - tbl_item.append({"data": None}) - continue + continue + except KeyError: + pass + except TypeError: + tbl_dict.pop(tst_name_mod, None) + if history: + for item in history: + for job, builds in item["data"].items(): + for build in builds: + for tst_name, tst_data in data[job][str(build)].iteritems(): + tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \ + replace("-ndrpdr", "").replace("-pdrdisc", ""). \ + replace("-ndrdisc", "").replace("-pdr", ""). \ + replace("-ndr", "").\ + replace("1t1c", "1c").replace("2t1c", "1c").\ + replace("2t2c", "2c").replace("4t2c", "2c").\ + replace("4t4c", "4c").replace("8t4c", "4c") + if "across topologies" in table["title"].lower(): + tst_name_mod = tst_name_mod.replace("2n1l-", "") + if tbl_dict.get(tst_name_mod, None) is None: + continue + if tbl_dict[tst_name_mod].get("history", None) is None: + tbl_dict[tst_name_mod]["history"] = OrderedDict() + if tbl_dict[tst_name_mod]["history"].get(item["title"], + None) is None: + tbl_dict[tst_name_mod]["history"][item["title"]] = \ + list() + try: + # TODO: Re-work when NDRPDRDISC tests are not used + if table["include-tests"] == "MRR": + tbl_dict[tst_name_mod]["history"][item["title" + ]].append(tst_data["result"]["receive-rate"]. + avg) + elif table["include-tests"] == "PDR": + if tst_data["type"] == "PDR": + tbl_dict[tst_name_mod]["history"][ + item["title"]].\ + append(tst_data["throughput"]["value"]) + elif tst_data["type"] == "NDRPDR": + tbl_dict[tst_name_mod]["history"][item[ + "title"]].append(tst_data["throughput"][ + "PDR"]["LOWER"]) + elif table["include-tests"] == "NDR": + if tst_data["type"] == "NDR": + tbl_dict[tst_name_mod]["history"][ + item["title"]].\ + append(tst_data["throughput"]["value"]) + elif tst_data["type"] == "NDRPDR": + tbl_dict[tst_name_mod]["history"][item[ + "title"]].append(tst_data["throughput"][ + "NDR"]["LOWER"]) + else: + continue + except (TypeError, KeyError): + pass + + tbl_lst = list() + for tst_name in tbl_dict.keys(): + item = [tbl_dict[tst_name]["name"], ] + if history: + if tbl_dict[tst_name].get("history", None) is not None: + for hist_data in tbl_dict[tst_name]["history"].values(): + if hist_data: + item.append(round(mean(hist_data) / 1000000, 2)) + item.append(round(stdev(hist_data) / 1000000, 2)) + else: + item.extend([None, None]) else: - logging.error("Not supported command {0}. Skipping the table.". - format(cmd)) - return None - tbl_lst.append(tbl_item) + item.extend([None, None]) + data_t = tbl_dict[tst_name]["ref-data"] + if data_t: + item.append(round(mean(data_t) / 1000000, 2)) + item.append(round(stdev(data_t) / 1000000, 2)) + else: + item.extend([None, None]) + data_t = tbl_dict[tst_name]["cmp-data"] + if data_t: + item.append(round(mean(data_t) / 1000000, 2)) + item.append(round(stdev(data_t) / 1000000, 2)) + else: + item.extend([None, None]) + if item[-4] is not None and item[-2] is not None and item[-4] != 0: + item.append(int(relative_change(float(item[-4]), float(item[-2])))) + if len(item) == len(header): + tbl_lst.append(item) # Sort the table according to the relative change - tbl_lst.sort(key=lambda rel: rel[-1]["data"], reverse=True) - - # Create the tables and write them to the files - file_names = [ - "{0}_ndr_top{1}".format(table["output-file"], table["output-file-ext"]), - "{0}_pdr_top{1}".format(table["output-file"], table["output-file-ext"]), - "{0}_ndr_low{1}".format(table["output-file"], table["output-file-ext"]), - "{0}_pdr_low{1}".format(table["output-file"], table["output-file-ext"]) - ] - - for file_name in file_names: - logging.info(" Writing the file '{0}'".format(file_name)) - with open(file_name, "w") as file_handler: - file_handler.write(",".join(header) + "\n") - for item in tbl_lst: - if isinstance(item[-1]["data"], float): - rel_change = round(item[-1]["data"], 1) - else: - rel_change = item[-1]["data"] - if "ndr_top" in file_name \ - and "ndr" in item[0]["data"] \ - and rel_change >= 10.0: - _write_line_to_file(file_handler, item) - elif "pdr_top" in file_name \ - and "pdr" in item[0]["data"] \ - and rel_change >= 10.0: - _write_line_to_file(file_handler, item) - elif "ndr_low" in file_name \ - and "ndr" in item[0]["data"] \ - and rel_change < 10.0: - _write_line_to_file(file_handler, item) - elif "pdr_low" in file_name \ - and "pdr" in item[0]["data"] \ - and rel_change < 10.0: - _write_line_to_file(file_handler, item) + tbl_lst.sort(key=lambda rel: rel[-1], reverse=True) - logging.info(" Done.") + # Generate csv tables: + csv_file = "{0}.csv".format(table["output-file"]) + with open(csv_file, "w") as file_handler: + file_handler.write(header_str) + for test in tbl_lst: + file_handler.write(",".join([str(item) for item in test]) + "\n") + convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"])) -def _read_csv_template(file_name): - """Read the template from a .csv file. - :param file_name: Name / full path / relative path of the file to read. - :type file_name: str - :returns: Data from the template as list (lines) of lists (items on line). - :rtype: list - :raises: PresentationError if it is not possible to read the file. +def table_nics_comparison(table, input_data): + """Generate the table(s) with algorithm: table_nics_comparison + specified in the specification file. + + :param table: Table to generate. + :param input_data: Data to process. + :type table: pandas.Series + :type input_data: InputData """ + logging.info(" Generating the table {0} ...". + format(table.get("title", ""))) + + # Transform the data + logging.info(" Creating the data set for the {0} '{1}'.". + format(table.get("type", ""), table.get("title", ""))) + data = input_data.filter_data(table, continue_on_error=True) + + # Prepare the header of the tables try: - with open(file_name, 'r') as csv_file: - tmpl_data = list() - for line in csv_file: - tmpl_data.append(line[:-1].split(",")) - return tmpl_data - except IOError as err: - raise PresentationError(str(err), level="ERROR") + header = ["Test case", ] + if table["include-tests"] == "MRR": + hdr_param = "Receive Rate" + else: + hdr_param = "Throughput" + + header.extend( + ["{0} {1} [Mpps]".format(table["reference"]["title"], hdr_param), + "{0} Stdev [Mpps]".format(table["reference"]["title"]), + "{0} {1} [Mpps]".format(table["compare"]["title"], hdr_param), + "{0} Stdev [Mpps]".format(table["compare"]["title"]), + "Delta [%]"]) + header_str = ",".join(header) + "\n" + except (AttributeError, KeyError) as err: + logging.error("The model is invalid, missing parameter: {0}". + format(err)) + return -def table_performance_comparison(table, input_data): - """Generate the table(s) with algorithm: table_performance_comparison + # Prepare data to the table: + tbl_dict = dict() + for job, builds in table["data"].items(): + for build in builds: + for tst_name, tst_data in data[job][str(build)].iteritems(): + tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\ + replace("-ndrpdr", "").replace("-pdrdisc", "").\ + replace("-ndrdisc", "").replace("-pdr", "").\ + replace("-ndr", "").\ + replace("1t1c", "1c").replace("2t1c", "1c").\ + replace("2t2c", "2c").replace("4t2c", "2c").\ + replace("4t4c", "4c").replace("8t4c", "4c") + tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod) + if tbl_dict.get(tst_name_mod, None) is None: + name = "-".join(tst_data["name"].split("-")[:-1]) + tbl_dict[tst_name_mod] = {"name": name, + "ref-data": list(), + "cmp-data": list()} + try: + if table["include-tests"] == "MRR": + result = tst_data["result"]["receive-rate"].avg + elif table["include-tests"] == "PDR": + result = tst_data["throughput"]["PDR"]["LOWER"] + elif table["include-tests"] == "NDR": + result = tst_data["throughput"]["NDR"]["LOWER"] + else: + result = None + + if result: + if table["reference"]["nic"] in tst_data["tags"]: + tbl_dict[tst_name_mod]["ref-data"].append(result) + elif table["compare"]["nic"] in tst_data["tags"]: + tbl_dict[tst_name_mod]["cmp-data"].append(result) + except (TypeError, KeyError) as err: + logging.debug("No data for {0}".format(tst_name)) + logging.debug(repr(err)) + # No data in output.xml for this test + + tbl_lst = list() + for tst_name in tbl_dict.keys(): + item = [tbl_dict[tst_name]["name"], ] + data_t = tbl_dict[tst_name]["ref-data"] + if data_t: + item.append(round(mean(data_t) / 1000000, 2)) + item.append(round(stdev(data_t) / 1000000, 2)) + else: + item.extend([None, None]) + data_t = tbl_dict[tst_name]["cmp-data"] + if data_t: + item.append(round(mean(data_t) / 1000000, 2)) + item.append(round(stdev(data_t) / 1000000, 2)) + else: + item.extend([None, None]) + if item[-4] is not None and item[-2] is not None and item[-4] != 0: + item.append(int(relative_change(float(item[-4]), float(item[-2])))) + if len(item) == len(header): + tbl_lst.append(item) + + # Sort the table according to the relative change + tbl_lst.sort(key=lambda rel: rel[-1], reverse=True) + + # Generate csv tables: + csv_file = "{0}.csv".format(table["output-file"]) + with open(csv_file, "w") as file_handler: + file_handler.write(header_str) + for test in tbl_lst: + file_handler.write(",".join([str(item) for item in test]) + "\n") + + convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"])) + + +def table_soak_vs_ndr(table, input_data): + """Generate the table(s) with algorithm: table_soak_vs_ndr specified in the specification file. :param table: Table to generate. @@ -355,183 +539,118 @@ def table_performance_comparison(table, input_data): format(table.get("title", ""))) # Transform the data - data = input_data.filter_data(table) + logging.info(" Creating the data set for the {0} '{1}'.". + format(table.get("type", ""), table.get("title", ""))) + data = input_data.filter_data(table, continue_on_error=True) - # Prepare the header of the tables + # Prepare the header of the table try: - header = ["Test case", - "{0} Throughput [Mpps]".format(table["reference"]["title"]), - "{0} stdev [Mpps]".format(table["reference"]["title"]), - "{0} Throughput [Mpps]".format(table["compare"]["title"]), - "{0} stdev [Mpps]".format(table["compare"]["title"]), - "Change [%]"] + header = [ + "Test case", + "{0} Throughput [Mpps]".format(table["reference"]["title"]), + "{0} Stdev [Mpps]".format(table["reference"]["title"]), + "{0} Throughput [Mpps]".format(table["compare"]["title"]), + "{0} Stdev [Mpps]".format(table["compare"]["title"]), + "Delta [%]", "Stdev of delta [%]"] header_str = ",".join(header) + "\n" except (AttributeError, KeyError) as err: logging.error("The model is invalid, missing parameter: {0}". format(err)) return - # Prepare data to the table: + # Create a list of available SOAK test results: tbl_dict = dict() - for job, builds in table["reference"]["data"].items(): + for job, builds in table["compare"]["data"].items(): for build in builds: for tst_name, tst_data in data[job][str(build)].iteritems(): - if tbl_dict.get(tst_name, None) is None: - name = "{0}-{1}".format(tst_data["parent"].split("-")[0], - "-".join(tst_data["name"]. - split("-")[1:])) - tbl_dict[tst_name] = {"name": name, - "ref-data": list(), - "cmp-data": list()} - try: - tbl_dict[tst_name]["ref-data"].\ - append(tst_data["throughput"]["value"]) - except TypeError: - pass # No data in output.xml for this test + if tst_data["type"] == "SOAK": + tst_name_mod = tst_name.replace("-soak", "") + if tbl_dict.get(tst_name_mod, None) is None: + groups = re.search(REGEX_NIC, tst_data["parent"]) + nic = groups.group(0) if groups else "" + name = "{0}-{1}".format(nic, "-".join(tst_data["name"]. + split("-")[:-1])) + tbl_dict[tst_name_mod] = { + "name": name, + "ref-data": list(), + "cmp-data": list() + } + try: + tbl_dict[tst_name_mod]["cmp-data"].append( + tst_data["throughput"]["LOWER"]) + except (KeyError, TypeError): + pass + tests_lst = tbl_dict.keys() - for job, builds in table["compare"]["data"].items(): + # Add corresponding NDR test results: + for job, builds in table["reference"]["data"].items(): for build in builds: for tst_name, tst_data in data[job][str(build)].iteritems(): - try: - tbl_dict[tst_name]["cmp-data"].\ - append(tst_data["throughput"]["value"]) - except KeyError: - pass - except TypeError: - tbl_dict.pop(tst_name, None) + tst_name_mod = tst_name.replace("-ndrpdr", "").\ + replace("-mrr", "") + if tst_name_mod in tests_lst: + try: + if tst_data["type"] in ("NDRPDR", "MRR", "BMRR"): + if table["include-tests"] == "MRR": + result = tst_data["result"]["receive-rate"].avg + elif table["include-tests"] == "PDR": + result = tst_data["throughput"]["PDR"]["LOWER"] + elif table["include-tests"] == "NDR": + result = tst_data["throughput"]["NDR"]["LOWER"] + else: + result = None + if result is not None: + tbl_dict[tst_name_mod]["ref-data"].append( + result) + except (KeyError, TypeError): + continue tbl_lst = list() for tst_name in tbl_dict.keys(): item = [tbl_dict[tst_name]["name"], ] - if tbl_dict[tst_name]["ref-data"]: - data_t = remove_outliers(tbl_dict[tst_name]["ref-data"], - table["outlier-const"]) - item.append(round(mean(data_t) / 1000000, 2)) - item.append(round(stdev(data_t) / 1000000, 2)) + data_r = tbl_dict[tst_name]["ref-data"] + if data_r: + data_r_mean = mean(data_r) + item.append(round(data_r_mean / 1000000, 2)) + data_r_stdev = stdev(data_r) + item.append(round(data_r_stdev / 1000000, 2)) else: + data_r_mean = None + data_r_stdev = None item.extend([None, None]) - if tbl_dict[tst_name]["cmp-data"]: - data_t = remove_outliers(tbl_dict[tst_name]["cmp-data"], - table["outlier-const"]) - item.append(round(mean(data_t) / 1000000, 2)) - item.append(round(stdev(data_t) / 1000000, 2)) + data_c = tbl_dict[tst_name]["cmp-data"] + if data_c: + data_c_mean = mean(data_c) + item.append(round(data_c_mean / 1000000, 2)) + data_c_stdev = stdev(data_c) + item.append(round(data_c_stdev / 1000000, 2)) else: + data_c_mean = None + data_c_stdev = None item.extend([None, None]) - if item[1] is not None and item[3] is not None: - item.append(int(relative_change(float(item[1]), float(item[3])))) - if len(item) == 6: + if data_r_mean and data_c_mean: + delta, d_stdev = relative_change_stdev( + data_r_mean, data_c_mean, data_r_stdev, data_c_stdev) + item.append(round(delta, 2)) + item.append(round(d_stdev, 2)) tbl_lst.append(item) # Sort the table according to the relative change tbl_lst.sort(key=lambda rel: rel[-1], reverse=True) - # Generate tables: - # All tests in csv: - tbl_names = ["{0}-ndr-1t1c-full{1}".format(table["output-file"], - table["output-file-ext"]), - "{0}-ndr-2t2c-full{1}".format(table["output-file"], - table["output-file-ext"]), - "{0}-ndr-4t4c-full{1}".format(table["output-file"], - table["output-file-ext"]), - "{0}-pdr-1t1c-full{1}".format(table["output-file"], - table["output-file-ext"]), - "{0}-pdr-2t2c-full{1}".format(table["output-file"], - table["output-file-ext"]), - "{0}-pdr-4t4c-full{1}".format(table["output-file"], - table["output-file-ext"]) - ] - for file_name in tbl_names: - logging.info(" Writing file: '{0}'".format(file_name)) - with open(file_name, "w") as file_handler: - file_handler.write(header_str) - for test in tbl_lst: - if (file_name.split("-")[-3] in test[0] and # NDR vs PDR - file_name.split("-")[-2] in test[0]): # cores - test[0] = "-".join(test[0].split("-")[:-1]) - file_handler.write(",".join([str(item) for item in test]) + - "\n") - - # All tests in txt: - tbl_names_txt = ["{0}-ndr-1t1c-full.txt".format(table["output-file"]), - "{0}-ndr-2t2c-full.txt".format(table["output-file"]), - "{0}-ndr-4t4c-full.txt".format(table["output-file"]), - "{0}-pdr-1t1c-full.txt".format(table["output-file"]), - "{0}-pdr-2t2c-full.txt".format(table["output-file"]), - "{0}-pdr-4t4c-full.txt".format(table["output-file"]) - ] - - for i, txt_name in enumerate(tbl_names_txt): - txt_table = None - logging.info(" Writing file: '{0}'".format(txt_name)) - with open(tbl_names[i], 'rb') as csv_file: - csv_content = csv.reader(csv_file, delimiter=',', quotechar='"') - for row in csv_content: - if txt_table is None: - txt_table = prettytable.PrettyTable(row) - else: - txt_table.add_row(row) - txt_table.align["Test case"] = "l" - with open(txt_name, "w") as txt_file: - txt_file.write(str(txt_table)) - - # Selected tests in csv: - input_file = "{0}-ndr-1t1c-full{1}".format(table["output-file"], - table["output-file-ext"]) - with open(input_file, "r") as in_file: - lines = list() - for line in in_file: - lines.append(line) - - output_file = "{0}-ndr-1t1c-top{1}".format(table["output-file"], - table["output-file-ext"]) - logging.info(" Writing file: '{0}'".format(output_file)) - with open(output_file, "w") as out_file: - out_file.write(header_str) - for i, line in enumerate(lines[1:]): - if i == table["nr-of-tests-shown"]: - break - out_file.write(line) - - output_file = "{0}-ndr-1t1c-bottom{1}".format(table["output-file"], - table["output-file-ext"]) - logging.info(" Writing file: '{0}'".format(output_file)) - with open(output_file, "w") as out_file: - out_file.write(header_str) - for i, line in enumerate(lines[-1:0:-1]): - if i == table["nr-of-tests-shown"]: - break - out_file.write(line) - - input_file = "{0}-pdr-1t1c-full{1}".format(table["output-file"], - table["output-file-ext"]) - with open(input_file, "r") as in_file: - lines = list() - for line in in_file: - lines.append(line) - - output_file = "{0}-pdr-1t1c-top{1}".format(table["output-file"], - table["output-file-ext"]) - logging.info(" Writing file: '{0}'".format(output_file)) - with open(output_file, "w") as out_file: - out_file.write(header_str) - for i, line in enumerate(lines[1:]): - if i == table["nr-of-tests-shown"]: - break - out_file.write(line) - - output_file = "{0}-pdr-1t1c-bottom{1}".format(table["output-file"], - table["output-file-ext"]) - logging.info(" Writing file: '{0}'".format(output_file)) - with open(output_file, "w") as out_file: - out_file.write(header_str) - for i, line in enumerate(lines[-1:0:-1]): - if i == table["nr-of-tests-shown"]: - break - out_file.write(line) + # Generate csv tables: + csv_file = "{0}.csv".format(table["output-file"]) + with open(csv_file, "w") as file_handler: + file_handler.write(header_str) + for test in tbl_lst: + file_handler.write(",".join([str(item) for item in test]) + "\n") + + convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"])) def table_performance_trending_dashboard(table, input_data): - """Generate the table(s) with algorithm: table_performance_comparison + """Generate the table(s) with algorithm: + table_performance_trending_dashboard specified in the specification file. :param table: Table to generate. @@ -544,14 +663,18 @@ def table_performance_trending_dashboard(table, input_data): format(table.get("title", ""))) # Transform the data - data = input_data.filter_data(table) + logging.info(" Creating the data set for the {0} '{1}'.". + format(table.get("type", ""), table.get("title", ""))) + data = input_data.filter_data(table, continue_on_error=True) # Prepare the header of the tables - header = ["Test case", - "Thput trend [Mpps]", - "Anomaly [Mpps]", - "Change [%]", - "Classification"] + header = ["Test Case", + "Trend [Mpps]", + "Short-Term Change [%]", + "Long-Term Change [%]", + "Regressions [#]", + "Progressions [#]" + ] header_str = ",".join(header) + "\n" # Prepare data to the table: @@ -559,119 +682,231 @@ def table_performance_trending_dashboard(table, input_data): for job, builds in table["data"].items(): for build in builds: for tst_name, tst_data in data[job][str(build)].iteritems(): + if tst_name.lower() in table["ignore-list"]: + continue if tbl_dict.get(tst_name, None) is None: - name = "{0}-{1}".format(tst_data["parent"].split("-")[0], - "-".join(tst_data["name"]. - split("-")[1:])) - tbl_dict[tst_name] = {"name": name, - "data": list()} + groups = re.search(REGEX_NIC, tst_data["parent"]) + if not groups: + continue + nic = groups.group(0) + tbl_dict[tst_name] = { + "name": "{0}-{1}".format(nic, tst_data["name"]), + "data": OrderedDict()} try: - tbl_dict[tst_name]["data"]. \ - append(tst_data["result"]["throughput"]) + tbl_dict[tst_name]["data"][str(build)] = \ + tst_data["result"]["receive-rate"] except (TypeError, KeyError): pass # No data in output.xml for this test tbl_lst = list() for tst_name in tbl_dict.keys(): - if len(tbl_dict[tst_name]["data"]) > 2: - sample_lst = tbl_dict[tst_name]["data"] - pd_data = pd.Series(sample_lst) - win_size = pd_data.size \ - if pd_data.size < table["window"] else table["window"] - # Test name: - name = tbl_dict[tst_name]["name"] - - # Trend list: - trend_lst = list(pd_data.rolling(window=win_size, min_periods=2). - median()) - # Stdevs list: - t_data, _ = find_outliers(pd_data) - t_data_lst = list(t_data) - stdev_lst = list(t_data.rolling(window=win_size, min_periods=2). - std()) - - rel_change_lst = [None, ] - classification_lst = [None, ] - for idx in range(1, len(trend_lst)): - # Relative changes list: - if not isnan(sample_lst[idx]) \ - and not isnan(trend_lst[idx])\ - and trend_lst[idx] != 0: - rel_change_lst.append( - int(relative_change(float(trend_lst[idx]), - float(sample_lst[idx])))) - else: - rel_change_lst.append(None) - # Classification list: - if isnan(t_data_lst[idx]) or isnan(stdev_lst[idx]): - classification_lst.append("outlier") - elif sample_lst[idx] < (trend_lst[idx] - 3*stdev_lst[idx]): - classification_lst.append("regression") - elif sample_lst[idx] > (trend_lst[idx] + 3*stdev_lst[idx]): - classification_lst.append("progression") - else: - classification_lst.append("normal") - - last_idx = len(sample_lst) - 1 - first_idx = last_idx - int(table["evaluated-window"]) - if first_idx < 0: - first_idx = 0 - - if "regression" in classification_lst[first_idx:]: - classification = "regression" - elif "outlier" in classification_lst[first_idx:]: - classification = "outlier" - elif "progression" in classification_lst[first_idx:]: - classification = "progression" - else: - classification = "normal" - - idx = len(classification_lst) - 1 - while idx: - if classification_lst[idx] == classification: - break - idx -= 1 - - trend = round(float(trend_lst[-2]) / 1000000, 2) \ - if not isnan(trend_lst[-2]) else '' - sample = round(float(sample_lst[idx]) / 1000000, 2) \ - if not isnan(sample_lst[idx]) else '' - rel_change = rel_change_lst[idx] \ - if rel_change_lst[idx] is not None else '' - tbl_lst.append([name, - trend, - sample, - rel_change, - classification]) - - # Sort the table according to the classification + data_t = tbl_dict[tst_name]["data"] + if len(data_t) < 2: + continue + + classification_lst, avgs = classify_anomalies(data_t) + + win_size = min(len(data_t), table["window"]) + long_win_size = min(len(data_t), table["long-trend-window"]) + + try: + max_long_avg = max( + [x for x in avgs[-long_win_size:-win_size] + if not isnan(x)]) + except ValueError: + max_long_avg = nan + last_avg = avgs[-1] + avg_week_ago = avgs[max(-win_size, -len(avgs))] + + if isnan(last_avg) or isnan(avg_week_ago) or avg_week_ago == 0.0: + rel_change_last = nan + else: + rel_change_last = round( + ((last_avg - avg_week_ago) / avg_week_ago) * 100, 2) + + if isnan(max_long_avg) or isnan(last_avg) or max_long_avg == 0.0: + rel_change_long = nan + else: + rel_change_long = round( + ((last_avg - max_long_avg) / max_long_avg) * 100, 2) + + if classification_lst: + if isnan(rel_change_last) and isnan(rel_change_long): + continue + if (isnan(last_avg) or + isnan(rel_change_last) or + isnan(rel_change_long)): + continue + tbl_lst.append( + [tbl_dict[tst_name]["name"], + round(last_avg / 1000000, 2), + rel_change_last, + rel_change_long, + classification_lst[-win_size:].count("regression"), + classification_lst[-win_size:].count("progression")]) + + tbl_lst.sort(key=lambda rel: rel[0]) + tbl_sorted = list() - for classification in ("regression", "outlier", "progression", "normal"): - tbl_tmp = [item for item in tbl_lst if item[4] == classification] - tbl_tmp.sort(key=lambda rel: rel[0]) - tbl_sorted.extend(tbl_tmp) + for nrr in range(table["window"], -1, -1): + tbl_reg = [item for item in tbl_lst if item[4] == nrr] + for nrp in range(table["window"], -1, -1): + tbl_out = [item for item in tbl_reg if item[5] == nrp] + tbl_out.sort(key=lambda rel: rel[2]) + tbl_sorted.extend(tbl_out) file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"]) - logging.info(" Writing file: '{0}'".format(file_name)) + logging.info(" Writing file: '{0}'".format(file_name)) with open(file_name, "w") as file_handler: file_handler.write(header_str) for test in tbl_sorted: file_handler.write(",".join([str(item) for item in test]) + '\n') txt_file_name = "{0}.txt".format(table["output-file"]) - txt_table = None - logging.info(" Writing file: '{0}'".format(txt_file_name)) - with open(file_name, 'rb') as csv_file: - csv_content = csv.reader(csv_file, delimiter=',', quotechar='"') - for row in csv_content: - if txt_table is None: - txt_table = prettytable.PrettyTable(row) + logging.info(" Writing file: '{0}'".format(txt_file_name)) + convert_csv_to_pretty_txt(file_name, txt_file_name) + + +def _generate_url(base, testbed, test_name): + """Generate URL to a trending plot from the name of the test case. + + :param base: The base part of URL common to all test cases. + :param testbed: The testbed used for testing. + :param test_name: The name of the test case. + :type base: str + :type testbed: str + :type test_name: str + :returns: The URL to the plot with the trending data for the given test + case. + :rtype str + """ + + url = base + file_name = "" + anchor = ".html#" + feature = "" + + if "lbdpdk" in test_name or "lbvpp" in test_name: + file_name = "link_bonding" + + elif "114b" in test_name and "vhost" in test_name: + file_name = "vts" + + elif "testpmd" in test_name or "l3fwd" in test_name: + file_name = "dpdk" + + elif "memif" in test_name: + file_name = "container_memif" + feature = "-base" + + elif "srv6" in test_name: + file_name = "srv6" + + elif "vhost" in test_name: + if "l2xcbase" in test_name or "l2bdbasemaclrn" in test_name: + file_name = "vm_vhost_l2" + if "114b" in test_name: + feature = "" + elif "l2xcbase" in test_name and "x520" in test_name: + feature = "-base-l2xc" + elif "l2bdbasemaclrn" in test_name and "x520" in test_name: + feature = "-base-l2bd" else: - txt_table.add_row(row) - txt_table.align["Test case"] = "l" - with open(txt_file_name, "w") as txt_file: - txt_file.write(str(txt_table)) + feature = "-base" + elif "ip4base" in test_name: + file_name = "vm_vhost_ip4" + feature = "-base" + + elif "ipsecbasetnlsw" in test_name: + file_name = "ipsecsw" + feature = "-base-scale" + + elif "ipsec" in test_name: + file_name = "ipsec" + feature = "-base-scale" + + elif "ethip4lispip" in test_name or "ethip4vxlan" in test_name: + file_name = "ip4_tunnels" + feature = "-base" + + elif "ip4base" in test_name or "ip4scale" in test_name: + file_name = "ip4" + if "xl710" in test_name: + feature = "-base-scale-features" + elif "iacl" in test_name: + feature = "-features-iacl" + elif "oacl" in test_name: + feature = "-features-oacl" + elif "snat" in test_name or "cop" in test_name: + feature = "-features" + else: + feature = "-base-scale" + + elif "ip6base" in test_name or "ip6scale" in test_name: + file_name = "ip6" + feature = "-base-scale" + + elif "l2xcbase" in test_name or "l2xcscale" in test_name \ + or "l2bdbasemaclrn" in test_name or "l2bdscale" in test_name \ + or "l2dbbasemaclrn" in test_name or "l2dbscale" in test_name: + file_name = "l2" + if "macip" in test_name: + feature = "-features-macip" + elif "iacl" in test_name: + feature = "-features-iacl" + elif "oacl" in test_name: + feature = "-features-oacl" + else: + feature = "-base-scale" + + if "x520" in test_name: + nic = "x520-" + elif "x710" in test_name: + nic = "x710-" + elif "xl710" in test_name: + nic = "xl710-" + elif "xxv710" in test_name: + nic = "xxv710-" + elif "vic1227" in test_name: + nic = "vic1227-" + elif "vic1385" in test_name: + nic = "vic1385-" + else: + nic = "" + anchor += nic + + if "64b" in test_name: + framesize = "64b" + elif "78b" in test_name: + framesize = "78b" + elif "imix" in test_name: + framesize = "imix" + elif "9000b" in test_name: + framesize = "9000b" + elif "1518b" in test_name: + framesize = "1518b" + elif "114b" in test_name: + framesize = "114b" + else: + framesize = "" + anchor += framesize + '-' + + if "1t1c" in test_name: + anchor += "1t1c" + elif "2t2c" in test_name: + anchor += "2t2c" + elif "4t4c" in test_name: + anchor += "4t4c" + elif "2t1c" in test_name: + anchor += "2t1c" + elif "4t2c" in test_name: + anchor += "4t2c" + elif "8t4c" in test_name: + anchor += "8t4c" + + return url + file_name + '-' + testbed + '-' + nic + framesize + feature + \ + anchor + feature def table_performance_trending_dashboard_html(table, input_data): @@ -681,10 +916,16 @@ def table_performance_trending_dashboard_html(table, input_data): :param table: Table to generate. :param input_data: Data to process. - :type table: pandas.Series + :type table: dict :type input_data: InputData """ + testbed = table.get("testbed", None) + if testbed is None: + logging.error("The testbed is not defined for the table '{0}'.". + format(table.get("title", ""))) + return + logging.info(" Generating the table {0} ...". format(table.get("title", ""))) @@ -704,37 +945,261 @@ def table_performance_trending_dashboard_html(table, input_data): dashboard = ET.Element("table", attrib=dict(width="100%", border='0')) # Table header: - tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor="#6699ff")) + tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor="#7eade7")) for idx, item in enumerate(csv_lst[0]): - alignment = "left" if idx == 0 else "right" + alignment = "left" if idx == 0 else "center" th = ET.SubElement(tr, "th", attrib=dict(align=alignment)) th.text = item # Rows: + colors = {"regression": ("#ffcccc", "#ff9999"), + "progression": ("#c6ecc6", "#9fdf9f"), + "normal": ("#e9f1fb", "#d4e4f7")} for r_idx, row in enumerate(csv_lst[1:]): - background = "#D4E4F7" if r_idx % 2 else "white" + if int(row[4]): + color = "regression" + elif int(row[5]): + color = "progression" + else: + color = "normal" + background = colors[color][r_idx % 2] tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor=background)) # Columns: for c_idx, item in enumerate(row): alignment = "left" if c_idx == 0 else "center" td = ET.SubElement(tr, "td", attrib=dict(align=alignment)) - if c_idx == 4: - if item == "regression": - td.set("bgcolor", "#eca1a6") - elif item == "outlier": - td.set("bgcolor", "#d6cbd3") - elif item == "progression": - td.set("bgcolor", "#bdcebe") - td.text = item - + # Name: + if c_idx == 0: + url = _generate_url("../trending/", testbed, item) + ref = ET.SubElement(td, "a", attrib=dict(href=url)) + ref.text = item + else: + td.text = item try: with open(table["output-file"], 'w') as html_file: - logging.info(" Writing file: '{0}'". - format(table["output-file"])) + logging.info(" Writing file: '{0}'".format(table["output-file"])) html_file.write(".. raw:: html\n\n\t") html_file.write(ET.tostring(dashboard)) html_file.write("\n\t