X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=blobdiff_plain;f=resources%2Ftools%2Fpresentation%2Fgenerator_alerts.py;h=9a0a03a59fdd752b57da65de213d6b4a2e0ab032;hp=77302b6168797b1b870825d9e5ca13bfc9e36856;hb=03bf0dd6ea67ef2b1386733d0b2ce3489c6a7f3e;hpb=5ad9b364cbd45a0b25d73412b9777ac14df92b0a diff --git a/resources/tools/presentation/generator_alerts.py b/resources/tools/presentation/generator_alerts.py index 77302b6168..9a0a03a59f 100644 --- a/resources/tools/presentation/generator_alerts.py +++ b/resources/tools/presentation/generator_alerts.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018 Cisco and/or its affiliates. +# Copyright (c) 2021 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -11,15 +11,24 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Generator of alerts: +- failed tests +- regressions +- progressions +""" + + import smtplib import logging +import re +from difflib import SequenceMatcher from email.mime.text import MIMEText from email.mime.multipart import MIMEMultipart from os.path import isdir +from collections import OrderedDict, defaultdict -from utils import execute_command -from errors import PresentationError +from pal_errors import PresentationError class AlertingError(PresentationError): @@ -33,7 +42,7 @@ class AlertingError(PresentationError): - relevant data if there are any collected (optional parameter details). """ - def __init__(self, msg, details='', level="CRITICAL"): + def __init__(self, msg, details=u'', level=u"CRITICAL"): """Sets the exception message and the level. :param msg: Short description of the encountered problem. @@ -47,16 +56,16 @@ class AlertingError(PresentationError): :type level: str """ - super(AlertingError, self).__init__( - "Alerting: {0}".format(msg), details, level) + super(AlertingError, self).__init__(f"Alerting: {msg}", details, level) def __repr__(self): return ( - "AlertingError(msg={msg!r},details={dets!r},level={level!r})". - format(msg=self._msg, dets=self._details, level=self._level)) + f"AlertingError(msg={self._msg!r},details={self._details!r}," + f"level={self._level!r})" + ) -class Alerting(object): +class Alerting: """Class implementing the alerting mechanism. """ @@ -68,49 +77,60 @@ class Alerting(object): """ # Implemented alerts: - self._ALERTS = ("failed-tests", ) + self._implemented_alerts = (u"failed-tests", ) + + self._spec = spec + + self.error_msgs = list() + + try: + self._spec_alert = spec.alerting + except KeyError as err: + raise AlertingError( + u"Alerting is not configured, skipped.", repr(err), u"WARNING" + ) - self._spec = spec.alerting - self._path_failed_tests = spec.environment["paths"]["DIR[STATIC,VPP]"] + self._path_failed_tests = spec.environment[u"paths"][u"DIR[STATIC,VPP]"] # Verify and validate input specification: - self.configs = self._spec.get("configurations", None) + self.configs = self._spec_alert.get(u"configurations", None) if not self.configs: - raise AlertingError("No alert configuration is specified.") - for config_type, config_data in self.configs.iteritems(): - if config_type == "email": - if not config_data.get("server", None): - raise AlertingError("Parameter 'server' is missing.") - if not config_data.get("address-to", None): - raise AlertingError("Parameter 'address-to' (recipient) is " - "missing.") - if not config_data.get("address-from", None): - raise AlertingError("Parameter 'address-from' (sender) is " - "missing.") - elif config_type == "jenkins": - if not isdir(config_data.get("output-dir", "")): - raise AlertingError("Parameter 'output-dir' is " - "missing or it is not a directory.") - if not config_data.get("output-file", None): - raise AlertingError("Parameter 'output-file' is missing.") + raise AlertingError(u"No alert configuration is specified.") + for config_type, config_data in self.configs.items(): + if config_type == u"email": + if not config_data.get(u"server", None): + raise AlertingError(u"Parameter 'server' is missing.") + if not config_data.get(u"address-to", None): + raise AlertingError(u"Parameter 'address-to' (recipient) " + u"is missing.") + if not config_data.get(u"address-from", None): + raise AlertingError(u"Parameter 'address-from' (sender) is " + u"missing.") + elif config_type == u"jenkins": + if not isdir(config_data.get(u"output-dir", u"")): + raise AlertingError(u"Parameter 'output-dir' is " + u"missing or it is not a directory.") + if not config_data.get(u"output-file", None): + raise AlertingError(u"Parameter 'output-file' is missing.") else: - raise AlertingError("Alert of type '{0}' is not implemented.". - format(config_type)) + raise AlertingError( + f"Alert of type {config_type} is not implemented." + ) - self.alerts = self._spec.get("alerts", None) + self.alerts = self._spec_alert.get(u"alerts", None) if not self.alerts: - raise AlertingError("No alert is specified.") - for alert, alert_data in self.alerts.iteritems(): - if not alert_data.get("title", None): - raise AlertingError("Parameter 'title' is missing.") - if not alert_data.get("type", None) in self._ALERTS: - raise AlertingError("Parameter 'failed-tests' is missing or " - "incorrect.") - if not alert_data.get("way", None) in self.configs.keys(): - raise AlertingError("Parameter 'way' is missing or incorrect.") - if not alert_data.get("include", None): - raise AlertingError("Parameter 'include' is missing or the " - "list is empty.") + raise AlertingError(u"No alert is specified.") + for alert_data in self.alerts.values(): + if not alert_data.get(u"title", None): + raise AlertingError(u"Parameter 'title' is missing.") + if not alert_data.get(u"type", None) in self._implemented_alerts: + raise AlertingError(u"Parameter 'failed-tests' is missing or " + u"incorrect.") + if not alert_data.get(u"way", None) in self.configs.keys(): + raise AlertingError(u"Parameter 'way' is missing or incorrect.") + if not alert_data.get(u"include", None): + raise AlertingError(u"Parameter 'include' is missing or the " + u"list is empty.") def __str__(self): """Return string with human readable description of the alert. @@ -118,8 +138,7 @@ class Alerting(object): :returns: Readable description. :rtype: str """ - return "configs={configs}, alerts={alerts}".format( - configs=self.configs, alerts=self.alerts) + return f"configs={self.configs}, alerts={self.alerts}" def __repr__(self): """Return string executable as Python constructor call. @@ -127,28 +146,19 @@ class Alerting(object): :returns: Executable constructor call. :rtype: str """ - return "Alerting(spec={spec})".format( - spec=self._spec) + return f"Alerting(spec={self._spec})" def generate_alerts(self): """Generate alert(s) using specified way(s). """ - for alert, alert_data in self.alerts.iteritems(): - if alert_data["way"] == "email": - text, html = self._create_alert_message(alert_data) - conf = self.configs["email"] - self._send_email(server=conf["server"], - addr_from=conf["address-from"], - addr_to=conf["address-to"], - subject=alert_data["title"], - text=text, - html=html) - elif alert_data["way"] == "jenkins": - self._generate_files_for_jenkins(alert_data) + for alert_data in self.alerts.values(): + if alert_data[u"way"] == u"jenkins": + self._generate_email_body(alert_data) else: - raise AlertingError("Alert with way '{0}' is not implemented.". - format(alert_data["way"])) + raise AlertingError( + f"Alert with way {alert_data[u'way']} is not implemented." + ) @staticmethod def _send_email(server, addr_from, addr_to, subject, text=None, html=None): @@ -169,109 +179,320 @@ class Alerting(object): """ if not text and not html: - raise AlertingError("No text/data to send.") + raise AlertingError(u"No text/data to send.") - msg = MIMEMultipart('alternative') - msg['Subject'] = subject - msg['From'] = addr_from - msg['To'] = ", ".join(addr_to) + msg = MIMEMultipart(u'alternative') + msg[u'Subject'] = subject + msg[u'From'] = addr_from + msg[u'To'] = u", ".join(addr_to) if text: - msg.attach(MIMEText(text, 'plain')) + msg.attach(MIMEText(text, u'plain')) if html: - msg.attach(MIMEText(html, 'html')) + msg.attach(MIMEText(html, u'html')) smtp_server = None try: - logging.info("Trying to send alert '{0}' ...".format(subject)) - logging.debug("SMTP Server: {0}".format(server)) - logging.debug("From: {0}".format(addr_from)) - logging.debug("To: {0}".format(", ".join(addr_to))) - logging.debug("Message: {0}".format(msg.as_string())) + logging.info(f"Trying to send alert {subject} ...") + logging.debug(f"SMTP Server: {server}") + logging.debug(f"From: {addr_from}") + logging.debug(f"To: {u', '.join(addr_to)}") + logging.debug(f"Message: {msg.as_string()}") smtp_server = smtplib.SMTP(server) smtp_server.sendmail(addr_from, addr_to, msg.as_string()) except smtplib.SMTPException as err: - raise AlertingError("Not possible to send the alert via email.", + raise AlertingError(u"Not possible to send the alert via email.", str(err)) finally: if smtp_server: smtp_server.quit() - def _create_alert_message(self, alert): - """Create the message which is used in the generated alert. + def _get_compressed_failed_tests(self, alert, test_set, sort=True): + """Return the dictionary with compressed faild tests. The compression is + done by grouping the tests from the same area but with different NICs, + frame sizes and number of processor cores. + + For example, the failed tests: + 10ge2p1x520-64b-1c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr + 10ge2p1x520-64b-2c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr + 10ge2p1x520-64b-4c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr + 10ge2p1x520-imix-1c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr + 10ge2p1x520-imix-2c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr + 10ge2p1x520-imix-4c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr + + will be represented as: + ethip4udp-ip4scale4000-udpsrcscale15-nat44 \ + (10ge2p1x520, 64b, imix, 1c, 2c, 4c) + + Structure of returned data: + + { + "trimmed_TC_name_1": { + "nics": [], + "framesizes": [], + "cores": [] + } + ... + "trimmed_TC_name_N": { + "nics": [], + "framesizes": [], + "cores": [] + } + } - :param alert: Message is created for this alert. + :param alert: Files are created for this alert. + :param test_set: Specifies which set of tests will be included in the + result. Its name is the same as the name of file with failed tests. + :param sort: If True, the failed tests are sorted alphabetically. :type alert: dict - :returns: Message in the ASCII text and HTML format. - :rtype: tuple(str, str) + :type test_set: str + :type sort: bool + :returns: CSIT build number, VPP version, Number of passed tests, + Number of failed tests, Compressed failed tests. + :rtype: tuple(str, str, int, int, str, OrderedDict) """ - if alert["type"] == "failed-tests": - text = "" - html = "" - for item in alert["include"]: - file_name = "{path}/{name}".format( - path=self._path_failed_tests, name=item) - try: - with open("{0}.txt".format(file_name), 'r') as txt_file: - text += "{0}:\n\n".format( - item.replace("failed-tests-", "")) - text += txt_file.read() + "\n" * 2 - except IOError: - logging.error("Not possible to read the file '{0}.txt'.". - format(file_name)) - try: - with open("{0}.rst".format(file_name), 'r') as rst_file: - html += "

{0}:

".format( - item.replace("failed-tests-", "")) - html += rst_file.readlines()[2].\ - replace("../trending", alert.get("url", "")) - html += "
" * 3 - except IOError: - logging.error("Not possible to read the file '{0}.rst'.". - format(file_name)) - html += "" - else: - raise AlertingError("Alert of type '{0}' is not implemented.". - format(alert["type"])) - return text, html - - def _generate_files_for_jenkins(self, alert): + directory = self.configs[alert[u"way"]][u"output-dir"] + failed_tests = defaultdict(dict) + file_path = f"{directory}/{test_set}.txt" + version = u"" + try: + with open(file_path, u'r') as f_txt: + for idx, line in enumerate(f_txt): + if idx == 0: + build = line[:-1] + continue + if idx == 1: + version = line[:-1] + continue + if idx == 2: + passed = line[:-1] + continue + if idx == 3: + failed = line[:-1] + continue + if idx == 4: + minutes = int(line[:-1]) // 60000 + duration = f"{(minutes // 60):02d}:{(minutes % 60):02d}" + continue + try: + line, error_msg = line[:-1].split(u'###', maxsplit=1) + test = line.split(u'-') + name = u'-'.join(test[3:-1]) + except ValueError: + continue + + for e_msg in self.error_msgs: + if SequenceMatcher(None, e_msg, + error_msg).ratio() > 0.5: + error_msg = e_msg + break + if error_msg not in self.error_msgs: + self.error_msgs.append(error_msg) + + error_msg_index = self.error_msgs.index(error_msg) + + if failed_tests.get(name, {}).get(error_msg_index) is None: + failed_tests[name][error_msg_index] = \ + dict(nics=list(), + framesizes=list(), + cores=list()) + + if test[0] not in \ + failed_tests[name][error_msg_index][u"nics"]: + failed_tests[name][error_msg_index][u"nics"].\ + append(test[0]) + if test[1] not in \ + failed_tests[name][error_msg_index][u"framesizes"]: + failed_tests[name][error_msg_index][u"framesizes"].\ + append(test[1]) + check_core = test[2] + f"[{str(error_msg_index)}]" + if check_core not in \ + failed_tests[name][error_msg_index][u"cores"]: + failed_tests[name][error_msg_index][u"cores"].\ + append(test[2] + "[" + str(error_msg_index) + "]") + + except IOError: + logging.error(f"No such file or directory: {file_path}") + return None, None, None, None, None, None + if sort: + sorted_failed_tests = OrderedDict() + for key in sorted(failed_tests.keys()): + sorted_failed_tests[key] = failed_tests[key] + return build, version, passed, failed, duration, sorted_failed_tests + + return build, version, passed, failed, duration, failed_tests + + def _list_gressions(self, alert, idx, header, re_pro): + """Create a file with regressions or progressions for the test set + specified by idx. + + :param alert: Files are created for this alert. + :param idx: Index of the test set as it is specified in the + specification file. + :param header: The header of the list of [re|pro]gressions. + :param re_pro: 'regression' or 'progression'. + :type alert: dict + :type idx: int + :type header: str + :type re_pro: str + """ + + if re_pro not in (u"regressions", u"progressions"): + return + + in_file = ( + f"{self.configs[alert[u'way']][u'output-dir']}/" + f"{re_pro}-{alert[u'urls'][idx].split(u'/')[-1]}.txt" + ) + out_file = ( + f"{self.configs[alert[u'way']][u'output-dir']}/" + f"trending-{re_pro}.txt" + ) + + try: + with open(in_file, u'r') as txt_file: + file_content = txt_file.read() + with open(out_file, u'a+') as reg_file: + reg_file.write(header) + if file_content: + reg_file.write(file_content) + else: + reg_file.write(f"No {re_pro}") + except IOError as err: + logging.warning(repr(err)) + + def _generate_email_body(self, alert): """Create the file which is used in the generated alert. :param alert: Files are created for this alert. :type alert: dict """ - config = self.configs[alert["way"]] + if alert[u"type"] != u"failed-tests": + raise AlertingError( + f"Alert of type {alert[u'type']} is not implemented." + ) - if alert["type"] == "failed-tests": - text, html = self._create_alert_message(alert) - file_name = "{0}/{1}".format(config["output-dir"], - config["output-file"]) - logging.info("Writing the file '{0}.txt' ...".format(file_name)) - try: - with open("{0}.txt".format(file_name), 'w') as txt_file: - txt_file.write(text) - except IOError: - logging.error("Not possible to write the file '{0}.txt'.". - format(file_name)) - logging.info("Writing the file '{0}.html' ...".format(file_name)) + text = u"" + for idx, test_set in enumerate(alert.get(u"include", list())): + test_set_short = u"" + device = u"" try: - with open("{0}.html".format(file_name), 'w') as html_file: - html_file.write(html) - except IOError: - logging.error("Not possible to write the file '{0}.html'.". - format(file_name)) - - zip_file = config.get("zip-output", None) - if zip_file: - logging.info("Writing the file '{0}/{1}' ...". - format(config["output-dir"], zip_file)) - execute_command("tar czvf {dir}/{zip} {dir}/{input}.*".format( - dir=config["output-dir"], - zip=zip_file, - input=config["output-file"])) - else: - raise AlertingError("Alert of type '{0}' is not implemented.". - format(alert["type"])) + groups = re.search( + re.compile( + r'((vpp|dpdk)-\dn-(skx|clx|tsh|dnv|zn2|tx2)-.*)' + ), + test_set + ) + test_set_short = groups.group(1) + device = groups.group(2) + except (AttributeError, IndexError): + logging.error( + f"The test set {test_set} does not include information " + f"about test bed. Using empty string instead." + ) + build, version, passed, failed, duration, failed_tests = \ + self._get_compressed_failed_tests(alert, test_set) + if build is None: + text += ( + f"\n\nNo input data available for {test_set_short}. " + f"See CSIT job {alert[u'urls'][idx]} for more " + f"information.\n" + ) + continue + text += ( + f"\n\n{test_set_short}, " + f"{failed} tests failed, " + f"{passed} tests passed, " + f"duration: {duration}, " + f"CSIT build: {alert[u'urls'][idx]}/{build}, " + f"{device} version: {version}\n\n" + ) + + class MaxLens(): + """Class to store the max lengths of strings displayed in + failed tests list. + """ + def __init__(self, tst_name, nics, framesizes, cores): + """Initialisation. + + :param tst_name: Name of the test. + :param nics: NICs used in the test. + :param framesizes: Frame sizes used in the tests + :param cores: Cores used in th test. + """ + self.name = tst_name + self.nics = nics + self.frmsizes = framesizes + self.cores = cores + + max_len = MaxLens(0, 0, 0, 0) + + for test, message in failed_tests.items(): + for e_message, params in message.items(): + failed_tests[test][e_message][u"nics"] = \ + u" ".join(sorted(params[u"nics"])) + failed_tests[test][e_message][u"framesizes"] = \ + u" ".join(sorted(params[u"framesizes"])) + failed_tests[test][e_message][u"cores"] = \ + u" ".join(sorted(params[u"cores"])) + if len(test) > max_len.name: + max_len.name = len(test) + if len(failed_tests[test][e_message][u"nics"]) > \ + max_len.nics: + max_len.nics = \ + len(failed_tests[test][e_message][u"nics"]) + if len(failed_tests[test][e_message][u"framesizes"]) > \ + max_len.frmsizes: + max_len.frmsizes = \ + len(failed_tests[test][e_message][u"framesizes"]) + if len(failed_tests[test][e_message][u"cores"]) > \ + max_len.cores: + max_len.cores = \ + len(failed_tests[test][e_message][u"cores"]) + + for test, message in failed_tests.items(): + test_added = False + for e_message, params in message.items(): + if not test_added: + test_added = True + else: + test = "" + text += ( + f"{test + u' ' * (max_len.name - len(test))} " + f"{params[u'nics']}" + f"{u' ' * (max_len.nics - len(params[u'nics']))} " + f"{params[u'framesizes']}" + f"""{u' ' * (max_len.frmsizes + - len(params[u'framesizes']))} """ + f"{params[u'cores']}" + f"{u' ' * (max_len.cores - len(params[u'cores']))}\n" + ) + + gression_hdr = ( + f"\n\n{test_set_short}, " + f"CSIT build: {alert[u'urls'][idx]}/{build}, " + f"{device} version: {version}\n\n" + ) + # Add list of regressions: + self._list_gressions(alert, idx, gression_hdr, u"regressions") + + # Add list of progressions: + self._list_gressions(alert, idx, gression_hdr, u"progressions") + + text += f"\nFor detailed information visit: {alert[u'url-details']}\n" + file_name = f"{self.configs[alert[u'way']][u'output-dir']}/" \ + f"{self.configs[alert[u'way']][u'output-file']}" + logging.info(f"Writing the file {file_name}.txt ...") + + text += f"\n\nLegend:\n\n" + + for e_msg in self.error_msgs: + text += f"[{self.error_msgs.index(e_msg)}] - {e_msg}\n" + + try: + with open(f"{file_name}.txt", u'w') as txt_file: + txt_file.write(text) + except IOError: + logging.error(f"Not possible to write the file {file_name}.txt.")