From: Tibor Frank Date: Mon, 1 Apr 2019 14:31:38 +0000 (+0200) Subject: Trending: Alerts X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=commitdiff_plain;h=d45c3f997201a6f17b39b3e5ceb82b332852149f Trending: Alerts Change-Id: I1c744f4eeefe3082be02daa681bfc032d6db4d15 Signed-off-by: Tibor Frank --- diff --git a/resources/tools/presentation/generator_alerts.py b/resources/tools/presentation/generator_alerts.py index 83dfe2eb17..ce5d8035e9 100644 --- a/resources/tools/presentation/generator_alerts.py +++ b/resources/tools/presentation/generator_alerts.py @@ -17,6 +17,7 @@ import logging from email.mime.text import MIMEText from email.mime.multipart import MIMEMultipart from os.path import isdir +from collections import OrderedDict from utils import execute_command from errors import PresentationError @@ -151,6 +152,8 @@ class Alerting(object): text=text, html=html) elif alert_data["way"] == "jenkins": + self._generate_email_body(alert_data) + # TODO: Remove when not needed self._generate_files_for_jenkins(alert_data) else: raise AlertingError("Alert with way '{0}' is not implemented.". @@ -203,82 +206,181 @@ class Alerting(object): if smtp_server: smtp_server.quit() - def _create_alert_message(self, alert): - """Create the message which is used in the generated alert. + def _get_compressed_failed_tests(self, alert, test_set, sort=True): + """Return the dictionary with compressed faild tests. The compression is + done by grouping the tests from the same area but with different NICs, + frame sizes and number of processor cores. + + For example, the failed tests: + 10ge2p1x520-64b-1c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr + 10ge2p1x520-64b-2c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr + 10ge2p1x520-64b-4c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr + 10ge2p1x520-imix-1c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr + 10ge2p1x520-imix-2c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr + 10ge2p1x520-imix-4c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr + + will be represented as: + ethip4udp-ip4scale4000-udpsrcscale15-nat44 \ + (10ge2p1x520, 64b, imix, 1c, 2c, 4c) + + Structure of returned data: + + { + "trimmed_TC_name_1": { + "nics": [], + "framesizes": [], + "cores": [] + } + ... + "trimmed_TC_name_N": { + "nics": [], + "framesizes": [], + "cores": [] + } + } - :param alert: Message is created for this alert. + :param alert: Files are created for this alert. + :param test_set: Specifies which set of tests will be included in the + result. Its name is the same as the name of file with failed tests. + :param sort: If True, the failed tests are sorted alphabetically. :type alert: dict - :returns: Message in the ASCII text and HTML format. - :rtype: tuple(str, str) + :type test_set: str + :type sort: bool + :returns: CSIT build number, VPP version, Number of failed tests, + Compressed failed tests. + :rtype: tuple(str, str, int, OrderedDict) """ - if alert["type"] == "failed-tests": - text = "" - html = "" - for item in alert["include"]: - file_name = "{path}/{name}".format( - path=self._path_failed_tests, name=item) - try: - with open("{0}.txt".format(file_name), 'r') as txt_file: - text += "{0}:\n\n".format( - item.replace("failed-tests-", "")) - text += txt_file.read() + "\n" * 2 - except IOError: - logging.error("Not possible to read the file '{0}.txt'.". - format(file_name)) - try: - with open("{0}.rst".format(file_name), 'r') as rst_file: - html += "

{0}:

".format( - item.replace("failed-tests-", "")) - html += rst_file.readlines()[2].\ - replace("../trending", alert.get("url", "")) - html += "
" * 3 - except IOError: - logging.error("Not possible to read the file '{0}.rst'.". - format(file_name)) - html += "" + directory = self.configs[alert["way"]]["output-dir"] + failed_tests = OrderedDict() + version = "" + try: + with open("{0}/{1}.txt".format(directory, test_set), 'r') as f_txt: + for idx, line in enumerate(f_txt): + if idx == 0: + build = line[:-1] + continue + if idx == 1: + version = line[:-1] + continue + try: + test = line[:-1].split('-') + nic = test[0] + framesize = test[1] + cores = test[2] + name = '-'.join(test[3:-1]) + except IndexError: + continue + if failed_tests.get(name, None) is None: + failed_tests[name] = dict(nics=list(), + framesizes=list(), + cores=list()) + if nic not in failed_tests[name]["nics"]: + failed_tests[name]["nics"].append(nic) + if framesize not in failed_tests[name]["framesizes"]: + failed_tests[name]["framesizes"].append(framesize) + if cores not in failed_tests[name]["cores"]: + failed_tests[name]["cores"].append(cores) + except IOError as err: + logging.error(repr(err)) + return None, None, None, None + if sort: + sorted_failed_tests = OrderedDict() + keys = [k for k in failed_tests.keys()] + keys.sort() + for key in keys: + sorted_failed_tests[key] = failed_tests[key] + return build, version, idx-1, sorted_failed_tests else: + return build, version, idx-1, failed_tests + + def _generate_email_body(self, alert): + """Create the file which is used in the generated alert. + + :param alert: Files are created for this alert. + :type alert: dict + """ + + if alert["type"] != "failed-tests": raise AlertingError("Alert of type '{0}' is not implemented.". format(alert["type"])) - return text, html + + config = self.configs[alert["way"]] + + text = "" + for idx, test_set in enumerate(alert.get("include", [])): + build, version, nr, failed_tests = \ + self._get_compressed_failed_tests(alert, test_set) + if build is None: + continue + text += ("\n\n{topo}-{arch}, " + "{nr} tests failed, " + "CSIT build: {link}/{build}, " + "VPP version: {version}\n\n". + format(topo=test_set.split('-')[-2], + arch=test_set.split('-')[-1], + nr=nr, + link=alert["urls"][idx], + build=build, + version=version)) + max_len_name = 0 + max_len_nics = 0 + max_len_framesizes = 0 + max_len_cores = 0 + for name, params in failed_tests.items(): + failed_tests[name]["nics"] = ",".join(sorted(params["nics"])) + failed_tests[name]["framesizes"] = \ + ",".join(sorted(params["framesizes"])) + failed_tests[name]["cores"] = ",".join(sorted(params["cores"])) + if len(name) > max_len_name: + max_len_name = len(name) + if len(failed_tests[name]["nics"]) > max_len_nics: + max_len_nics = len(failed_tests[name]["nics"]) + if len(failed_tests[name]["framesizes"]) > max_len_framesizes: + max_len_framesizes = len(failed_tests[name]["framesizes"]) + if len(failed_tests[name]["cores"]) > max_len_cores: + max_len_cores = len(failed_tests[name]["cores"]) + + for name, params in failed_tests.items(): + text += "{name} {nics} {frames} {cores}\n".format( + name=name + " " * (max_len_name - len(name)), + nics=params["nics"] + + " " * (max_len_nics - len(params["nics"])), + frames=params["framesizes"] + " " * + (max_len_framesizes - len(params["framesizes"])), + cores=params["cores"] + + " " * (max_len_cores - len(params["cores"]))) + + text += "\nFor detailed information visit: {url}\n".\ + format(url=alert["url-details"]) + file_name = "{0}/{1}".format(config["output-dir"], + config["output-file"]) + logging.info("Writing the file '{0}.txt' ...".format(file_name)) + + try: + with open("{0}.txt".format(file_name), 'w') as txt_file: + txt_file.write(text) + except IOError: + logging.error("Not possible to write the file '{0}.txt'.". + format(file_name)) def _generate_files_for_jenkins(self, alert): """Create the file which is used in the generated alert. + # TODO: Remove when not needed. + :param alert: Files are created for this alert. :type alert: dict """ config = self.configs[alert["way"]] - if alert["type"] == "failed-tests": - text, html = self._create_alert_message(alert) - file_name = "{0}/{1}".format(config["output-dir"], - config["output-file"]) - logging.info("Writing the file '{0}.txt' ...".format(file_name)) - try: - with open("{0}.txt".format(file_name), 'w') as txt_file: - txt_file.write(text) - except IOError: - logging.error("Not possible to write the file '{0}.txt'.". - format(file_name)) - logging.info("Writing the file '{0}.html' ...".format(file_name)) - try: - with open("{0}.html".format(file_name), 'w') as html_file: - html_file.write(html) - except IOError: - logging.error("Not possible to write the file '{0}.html'.". - format(file_name)) - - zip_file = config.get("zip-output", None) - if zip_file: - logging.info("Writing the file '{0}/{1}' ...". - format(config["output-dir"], zip_file)) - execute_command("tar czvf {dir}/{zip} --directory={dir} " - "{input}.txt {input}.html". - format(dir=config["output-dir"], - zip=zip_file, - input=config["output-file"])) - else: - raise AlertingError("Alert of type '{0}' is not implemented.". - format(alert["type"])) + zip_file = config.get("zip-output", None) + if zip_file: + logging.info("Writing the file '{0}/{1}' ...". + format(config["output-dir"], zip_file)) + execute_command("tar czvf {dir}/{zip} --directory={dir} " + "{input}.txt". + format(dir=config["output-dir"], + zip=zip_file, + input=config["output-file"])) diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py index b55e4a545d..cdce5f98c0 100644 --- a/resources/tools/presentation/generator_tables.py +++ b/resources/tools/presentation/generator_tables.py @@ -858,6 +858,51 @@ def table_performance_trending_dashboard_html(table, input_data): return +def table_last_failed_tests(table, input_data): + """Generate the table(s) with algorithm: table_last_failed_tests + specified in the specification file. + + :param table: Table to generate. + :param input_data: Data to process. + :type table: pandas.Series + :type input_data: InputData + """ + + logging.info(" Generating the table {0} ...". + format(table.get("title", ""))) + + # Transform the data + logging.info(" Creating the data set for the {0} '{1}'.". + format(table.get("type", ""), table.get("title", ""))) + data = input_data.filter_data(table, continue_on_error=True) + + if data is None or data.empty: + logging.warn(" No data for the {0} '{1}'.". + format(table.get("type", ""), table.get("title", ""))) + return + + tbl_list = list() + for job, builds in table["data"].items(): + for build in builds: + build = str(build) + tbl_list.append(build) + tbl_list.append(input_data.metadata(job, build).get("version", "")) + for tst_name, tst_data in data[job][build].iteritems(): + if tst_data["status"] != "FAIL": + continue + groups = re.search(REGEX_NIC, tst_data["parent"]) + if not groups: + continue + nic = groups.group(0) + tbl_list.append("{0}-{1}".format(nic, tst_data["name"])) + + file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"]) + logging.info(" Writing file: '{0}'".format(file_name)) + with open(file_name, "w") as file_handler: + for test in tbl_list: + file_handler.write(test + '\n') + + def table_failed_tests(table, input_data): """Generate the table(s) with algorithm: table_failed_tests specified in the specification file. diff --git a/resources/tools/presentation/specification_CPTA.yaml b/resources/tools/presentation/specification_CPTA.yaml index fe89093369..bf1a359748 100644 --- a/resources/tools/presentation/specification_CPTA.yaml +++ b/resources/tools/presentation/specification_CPTA.yaml @@ -119,10 +119,16 @@ type: "failed-tests" way: "jenkins" include: - - "failed-tests-3n-hsw" - - "failed-tests-3n-skx" - - "failed-tests-2n-skx" - url: "https://docs.fd.io/csit/master/trending/trending" + - "last-failed-tests-3n-hsw" + - "last-failed-tests-3n-skx" + - "last-failed-tests-2n-skx" + - "last-failed-tests-nf-2n-skx" + urls: + - "https://jenkins.fd.io/view/csit/job/csit-vpp-perf-mrr-daily-master" + - "https://jenkins.fd.io/view/csit/job/csit-vpp-perf-mrr-daily-master-3n-skx" + - "https://jenkins.fd.io/view/csit/job/csit-vpp-perf-mrr-daily-master-2n-skx" + - "https://jenkins.fd.io/view/csit/job/csit-vpp-perf-mrr-weekly-master-2n-skx" + url-details: "https://docs.fd.io/csit/master/trending/introduction/failures.html" configurations: # Configuration of the email notifications. @@ -148,19 +154,49 @@ data-sets: + # Compressed failed tests (last builds) + table-last-failed-tests-3n-hsw: + csit-vpp-perf-mrr-daily-master: + - "lastCompletedBuild" + table-last-failed-tests-3n-skx: + csit-vpp-perf-mrr-daily-master-3n-skx: + - "lastCompletedBuild" + table-last-failed-tests-2n-skx: + csit-vpp-perf-mrr-daily-master-2n-skx: + - "lastCompletedBuild" + table-last-failed-tests-nf-2n-skx: + csit-vpp-perf-mrr-weekly-master-2n-skx: + - "lastCompletedBuild" + # 3n-hsw plot-performance-trending-all-3n-hsw: csit-vpp-perf-mrr-daily-master: - start: 482 + start: 487 end: "lastCompletedBuild" + skip: + - 593 + - 595 + - 608 + - 613 + - 614 + - 616 + - 617 csit-dpdk-perf-mrr-weekly-master: start: 55 end: "lastCompletedBuild" plot-performance-trending-vpp-3n-hsw: csit-vpp-perf-mrr-daily-master: - start: 482 + start: 487 end: "lastCompletedBuild" + skip: + - 593 + - 595 + - 608 + - 613 + - 614 + - 616 + - 617 plot-performance-trending-dpdk-3n-hsw: csit-dpdk-perf-mrr-weekly-master: @@ -170,7 +206,7 @@ # 3n-skx plot-performance-trending-all-3n-skx: csit-vpp-perf-mrr-daily-master-3n-skx: - start: 232 + start: 237 end: "lastCompletedBuild" skip: - 356 @@ -203,7 +239,7 @@ plot-performance-trending-vpp-3n-skx: csit-vpp-perf-mrr-daily-master-3n-skx: - start: 232 + start: 237 end: "lastCompletedBuild" skip: - 356 @@ -228,6 +264,7 @@ - 376 - 377 - 378 + plot-performance-trending-dpdk-3n-skx: csit-dpdk-perf-mrr-weekly-master-3n-skx: start: 20 @@ -238,16 +275,48 @@ # 2n-skx plot-performance-trending-all-2n-skx: csit-vpp-perf-mrr-daily-master-2n-skx: - start: 232 + start: 237 end: "lastCompletedBuild" + skip: + - 347 + - 358 + - 359 + - 360 + - 361 + - 362 + - 363 + - 364 + - 365 + - 366 + - 367 + - 368 + - 375 + - 380 + - 381 csit-dpdk-perf-mrr-weekly-master-2n-skx: start: 20 end: "lastCompletedBuild" plot-performance-trending-vpp-2n-skx: csit-vpp-perf-mrr-daily-master-2n-skx: - start: 232 + start: 237 end: "lastCompletedBuild" + skip: + - 347 + - 358 + - 359 + - 360 + - 361 + - 362 + - 363 + - 364 + - 365 + - 366 + - 367 + - 368 + - 375 + - 380 + - 381 plot-performance-trending-dpdk-2n-skx: csit-dpdk-perf-mrr-weekly-master-2n-skx: @@ -352,15 +421,23 @@ # 3n-hsw csit-vpp-perf-mrr-daily-master: - start: 482 + start: 487 end: "lastCompletedBuild" + skip: + - 593 + - 595 + - 608 + - 613 + - 614 + - 616 + - 617 csit-dpdk-perf-mrr-weekly-master: start: 55 end: "lastCompletedBuild" # 3n-skx csit-vpp-perf-mrr-daily-master-3n-skx: - start: 232 + start: 237 end: "lastCompletedBuild" skip: - 356 @@ -393,8 +470,24 @@ # 2n-skx csit-vpp-perf-mrr-daily-master-2n-skx: - start: 232 + start: 237 end: "lastCompletedBuild" + skip: + - 347 + - 358 + - 359 + - 360 + - 361 + - 362 + - 363 + - 364 + - 365 + - 366 + - 367 + - 368 + - 375 + - 380 + - 381 csit-vpp-perf-mrr-weekly-master-2n-skx: start: 1 end: "lastCompletedBuild" @@ -418,6 +511,60 @@ ################################################################################ ################################################################################ + +# Compressed failed tests (last build) +- + type: "table" + title: "Last failed tests (last builds) 3n-hsw" + algorithm: "table_last_failed_tests" + output-file-ext: ".txt" + output-file: "{DIR[STATIC,VPP]}/last-failed-tests-3n-hsw" + data: "table-last-failed-tests-3n-hsw" + filter: "'MRR'" + parameters: + - "name" + - "parent" + - "status" + +- + type: "table" + title: "Last failed tests (last builds) 3n-skx" + algorithm: "table_last_failed_tests" + output-file-ext: ".txt" + output-file: "{DIR[STATIC,VPP]}/last-failed-tests-3n-skx" + data: "table-last-failed-tests-3n-skx" + filter: "'MRR'" + parameters: + - "name" + - "parent" + - "status" + +- + type: "table" + title: "Last failed tests (last builds) 2n-skx" + algorithm: "table_last_failed_tests" + output-file-ext: ".txt" + output-file: "{DIR[STATIC,VPP]}/last-failed-tests-2n-skx" + data: "table-last-failed-tests-2n-skx" + filter: "'MRR'" + parameters: + - "name" + - "parent" + - "status" + +- + type: "table" + title: "Last failed tests (last builds) NF 2n-skx" + algorithm: "table_last_failed_tests" + output-file-ext: ".txt" + output-file: "{DIR[STATIC,VPP]}/last-failed-tests-nf-2n-skx" + data: "table-last-failed-tests-nf-2n-skx" + filter: "'MRR'" + parameters: + - "name" + - "parent" + - "status" + # 3n-hsw - type: "table" diff --git a/resources/tools/presentation/specification_parser.py b/resources/tools/presentation/specification_parser.py index ae566c67fa..53649609dd 100644 --- a/resources/tools/presentation/specification_parser.py +++ b/resources/tools/presentation/specification_parser.py @@ -434,6 +434,13 @@ class Specification(object): builds = [x for x in range(builds["start"], build_end+1) if x not in builds.get("skip", list())] self.configuration["data-sets"][set_name][job] = builds + elif isinstance(builds, list): + for idx, item in enumerate(builds): + try: + builds[idx] = int(item) + except ValueError: + # defined as a range + builds[idx] = self._get_build_number(job, item) # Data sets: add sub-sets to sets (only one level): for set_name, data_set in self.configuration["data-sets"].items(): diff --git a/resources/tools/presentation_new/specification_CPTA.yaml b/resources/tools/presentation_new/specification_CPTA.yaml index e72fd9aa43..b2e47fd4c4 100644 --- a/resources/tools/presentation_new/specification_CPTA.yaml +++ b/resources/tools/presentation_new/specification_CPTA.yaml @@ -84,7 +84,7 @@ # 3n-hsw plot-performance-trending-all-3n-hsw: csit-vpp-perf-mrr-daily-master: - start: 470 + start: 500 end: "lastCompletedBuild" csit-dpdk-perf-mrr-weekly-master: start: 54 @@ -92,7 +92,7 @@ plot-performance-trending-vpp-3n-hsw: csit-vpp-perf-mrr-daily-master: - start: 470 + start: 500 end: "lastCompletedBuild" plot-performance-trending-dpdk-3n-hsw: @@ -103,7 +103,7 @@ # 3n-skx plot-performance-trending-all-3n-skx: csit-vpp-perf-mrr-daily-master-3n-skx: - start: 220 + start: 250 end: "lastCompletedBuild" csit-dpdk-perf-mrr-weekly-master-3n-skx: start: 19 @@ -111,7 +111,7 @@ plot-performance-trending-vpp-3n-skx: csit-vpp-perf-mrr-daily-master-3n-skx: - start: 220 + start: 250 end: "lastCompletedBuild" plot-performance-trending-dpdk-3n-skx: @@ -122,7 +122,7 @@ # 2n-skx plot-performance-trending-all-2n-skx: csit-vpp-perf-mrr-daily-master-2n-skx: - start: 220 + start: 250 end: "lastCompletedBuild" csit-dpdk-perf-mrr-weekly-master-2n-skx: start: 19 @@ -130,7 +130,7 @@ plot-performance-trending-vpp-2n-skx: csit-vpp-perf-mrr-daily-master-2n-skx: - start: 220 + start: 250 end: "lastCompletedBuild" plot-performance-trending-dpdk-2n-skx: @@ -228,7 +228,7 @@ # 3n-hsw csit-vpp-perf-mrr-daily-master: - start: 470 + start: 500 end: "lastCompletedBuild" csit-dpdk-perf-mrr-weekly-master: start: 54 @@ -236,7 +236,7 @@ # 3n-skx csit-vpp-perf-mrr-daily-master-3n-skx: - start: 220 + start: 250 end: "lastCompletedBuild" csit-dpdk-perf-mrr-weekly-master-3n-skx: start: 19 @@ -244,7 +244,7 @@ # 2n-skx csit-vpp-perf-mrr-daily-master-2n-skx: - start: 220 + start: 250 end: "lastCompletedBuild" csit-dpdk-perf-mrr-weekly-master-2n-skx: start: 19