Report: Current vs. Previous Release - NFV Tests
[csit.git] / resources / tools / presentation / generator_alerts.py
index cbdb1fd..b0606b6 100644 (file)
@@ -1,4 +1,4 @@
-# Copyright (c) 2018 Cisco and/or its affiliates.
+# Copyright (c) 2019 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+"""Generator of alerts:
+- failed tests
+- regressions
+- progressions
+"""
+
+
 import smtplib
 import logging
 
 from email.mime.text import MIMEText
 from email.mime.multipart import MIMEMultipart
 from os.path import isdir
 import smtplib
 import logging
 
 from email.mime.text import MIMEText
 from email.mime.multipart import MIMEMultipart
 from os.path import isdir
+from collections import OrderedDict
 
 
-from utils import execute_command
-from errors import PresentationError
+from pal_utils import get_last_completed_build_number
+from pal_errors import PresentationError
 
 
 class AlertingError(PresentationError):
 
 
 class AlertingError(PresentationError):
@@ -33,7 +41,7 @@ class AlertingError(PresentationError):
      - relevant data if there are any collected (optional parameter details).
     """
 
      - relevant data if there are any collected (optional parameter details).
     """
 
-    def __init__(self, msg, details='', level="CRITICAL"):
+    def __init__(self, msg, details=u'', level=u"CRITICAL"):
         """Sets the exception message and the level.
 
         :param msg: Short description of the encountered problem.
         """Sets the exception message and the level.
 
         :param msg: Short description of the encountered problem.
@@ -47,16 +55,16 @@ class AlertingError(PresentationError):
         :type level: str
         """
 
         :type level: str
         """
 
-        super(AlertingError, self).__init__(
-            "Alerting: {0}".format(msg), details, level)
+        super(AlertingError, self).__init__(f"Alerting: {msg}", details, level)
 
     def __repr__(self):
         return (
 
     def __repr__(self):
         return (
-            "AlertingError(msg={msg!r},details={dets!r},level={level!r})".
-            format(msg=self._msg, dets=self._details, level=self._level))
+            f"AlertingError(msg={self._msg!r},details={self._details!r},"
+            f"level={self._level!r})"
+        )
 
 
 
 
-class Alerting(object):
+class Alerting:
     """Class implementing the alerting mechanism.
     """
 
     """Class implementing the alerting mechanism.
     """
 
@@ -68,49 +76,58 @@ class Alerting(object):
         """
 
         # Implemented alerts:
         """
 
         # Implemented alerts:
-        self._ALERTS = ("failed-tests", )
+        self._implemented_alerts = (u"failed-tests", )
+
+        self._spec = spec
+
+        try:
+            self._spec_alert = spec.alerting
+        except KeyError as err:
+            raise AlertingError(u"Alerting is not configured, skipped.",
+                                repr(err),
+                                u"WARNING")
 
 
-        self._spec = spec.alerting
-        self._path_failed_tests = spec.environment["paths"]["DIR[STATIC,VPP]"]
+        self._path_failed_tests = spec.environment[u"paths"][u"DIR[STATIC,VPP]"]
 
         # Verify and validate input specification:
 
         # Verify and validate input specification:
-        self.configs = self._spec.get("configurations", None)
+        self.configs = self._spec_alert.get(u"configurations", None)
         if not self.configs:
         if not self.configs:
-            raise AlertingError("No alert configuration is specified.")
-        for config_type, config_data in self.configs.iteritems():
-            if config_type == "email":
-                if not config_data.get("server", None):
-                    raise AlertingError("Parameter 'server' is missing.")
-                if not config_data.get("address-to", None):
-                    raise AlertingError("Parameter 'address-to' (recipient) is "
-                                        "missing.")
-                if not config_data.get("address-from", None):
-                    raise AlertingError("Parameter 'address-from' (sender) is "
-                                        "missing.")
-            elif config_type == "jenkins":
-                if not isdir(config_data.get("output-dir", "")):
-                    raise AlertingError("Parameter 'output-dir' is "
-                                        "missing or it is not a directory.")
-                if not config_data.get("output-file", None):
-                    raise AlertingError("Parameter 'output-file' is missing.")
+            raise AlertingError(u"No alert configuration is specified.")
+        for config_type, config_data in self.configs.items():
+            if config_type == u"email":
+                if not config_data.get(u"server", None):
+                    raise AlertingError(u"Parameter 'server' is missing.")
+                if not config_data.get(u"address-to", None):
+                    raise AlertingError(u"Parameter 'address-to' (recipient) "
+                                        u"is missing.")
+                if not config_data.get(u"address-from", None):
+                    raise AlertingError(u"Parameter 'address-from' (sender) is "
+                                        u"missing.")
+            elif config_type == u"jenkins":
+                if not isdir(config_data.get(u"output-dir", u"")):
+                    raise AlertingError(u"Parameter 'output-dir' is "
+                                        u"missing or it is not a directory.")
+                if not config_data.get(u"output-file", None):
+                    raise AlertingError(u"Parameter 'output-file' is missing.")
             else:
             else:
-                raise AlertingError("Alert of type '{0}' is not implemented.".
-                                    format(config_type))
+                raise AlertingError(
+                    f"Alert of type {config_type} is not implemented."
+                )
 
 
-        self.alerts = self._spec.get("alerts", None)
+        self.alerts = self._spec_alert.get(u"alerts", None)
         if not self.alerts:
         if not self.alerts:
-            raise AlertingError("No alert is specified.")
-        for alert, alert_data in self.alerts.iteritems():
-            if not alert_data.get("title", None):
-                raise AlertingError("Parameter 'title' is missing.")
-            if not alert_data.get("type", None) in self._ALERTS:
-                raise AlertingError("Parameter 'failed-tests' is missing or "
-                                    "incorrect.")
-            if not alert_data.get("way", None) in self.configs.keys():
-                raise AlertingError("Parameter 'way' is missing or incorrect.")
-            if not alert_data.get("include", None):
-                raise AlertingError("Parameter 'include' is missing or the "
-                                    "list is empty.")
+            raise AlertingError(u"No alert is specified.")
+        for alert_data in self.alerts.values():
+            if not alert_data.get(u"title", None):
+                raise AlertingError(u"Parameter 'title' is missing.")
+            if not alert_data.get(u"type", None) in self._implemented_alerts:
+                raise AlertingError(u"Parameter 'failed-tests' is missing or "
+                                    u"incorrect.")
+            if not alert_data.get(u"way", None) in self.configs.keys():
+                raise AlertingError(u"Parameter 'way' is missing or incorrect.")
+            if not alert_data.get(u"include", None):
+                raise AlertingError(u"Parameter 'include' is missing or the "
+                                    u"list is empty.")
 
     def __str__(self):
         """Return string with human readable description of the alert.
 
     def __str__(self):
         """Return string with human readable description of the alert.
@@ -118,8 +135,7 @@ class Alerting(object):
         :returns: Readable description.
         :rtype: str
         """
         :returns: Readable description.
         :rtype: str
         """
-        return "configs={configs}, alerts={alerts}".format(
-            configs=self.configs, alerts=self.alerts)
+        return f"configs={self.configs}, alerts={self.alerts}"
 
     def __repr__(self):
         """Return string executable as Python constructor call.
 
     def __repr__(self):
         """Return string executable as Python constructor call.
@@ -127,28 +143,19 @@ class Alerting(object):
         :returns: Executable constructor call.
         :rtype: str
         """
         :returns: Executable constructor call.
         :rtype: str
         """
-        return "Alerting(spec={spec})".format(
-            spec=self._spec)
+        return f"Alerting(spec={self._spec})"
 
     def generate_alerts(self):
         """Generate alert(s) using specified way(s).
         """
 
 
     def generate_alerts(self):
         """Generate alert(s) using specified way(s).
         """
 
-        for alert, alert_data in self.alerts.iteritems():
-            if alert_data["way"] == "email":
-                text, html = self._create_alert_message(alert_data)
-                conf = self.configs["email"]
-                self._send_email(server=conf["server"],
-                                 addr_from=conf["address-from"],
-                                 addr_to=conf["address-to"],
-                                 subject=alert_data["title"],
-                                 text=text,
-                                 html=html)
-            elif alert_data["way"] == "jenkins":
-                self._generate_files_for_jenkins(alert_data)
+        for alert_data in self.alerts.values():
+            if alert_data[u"way"] == u"jenkins":
+                self._generate_email_body(alert_data)
             else:
             else:
-                raise AlertingError("Alert with way '{0}' is not implemented.".
-                                    format(alert_data["way"]))
+                raise AlertingError(
+                    f"Alert with way {alert_data[u'way']} is not implemented."
+                )
 
     @staticmethod
     def _send_email(server, addr_from, addr_to, subject, text=None, html=None):
 
     @staticmethod
     def _send_email(server, addr_from, addr_to, subject, text=None, html=None):
@@ -169,110 +176,261 @@ class Alerting(object):
         """
 
         if not text and not html:
         """
 
         if not text and not html:
-            raise AlertingError("No text/data to send.")
+            raise AlertingError(u"No text/data to send.")
 
 
-        msg = MIMEMultipart('alternative')
-        msg['Subject'] = subject
-        msg['From'] = addr_from
-        msg['To'] = ", ".join(addr_to)
+        msg = MIMEMultipart(u'alternative')
+        msg[u'Subject'] = subject
+        msg[u'From'] = addr_from
+        msg[u'To'] = u", ".join(addr_to)
 
         if text:
 
         if text:
-            msg.attach(MIMEText(text, 'plain'))
+            msg.attach(MIMEText(text, u'plain'))
         if html:
         if html:
-            msg.attach(MIMEText(html, 'html'))
+            msg.attach(MIMEText(html, u'html'))
 
         smtp_server = None
         try:
 
         smtp_server = None
         try:
-            logging.info("Trying to send alert '{0}' ...".format(subject))
-            logging.debug("SMTP Server: {0}".format(server))
-            logging.debug("From: {0}".format(addr_from))
-            logging.debug("To: {0}".format(", ".join(addr_to)))
-            logging.debug("Message: {0}".format(msg.as_string()))
+            logging.info(f"Trying to send alert {subject} ...")
+            logging.debug(f"SMTP Server: {server}")
+            logging.debug(f"From: {addr_from}")
+            logging.debug(f"To: {u', '.join(addr_to)}")
+            logging.debug(f"Message: {msg.as_string()}")
             smtp_server = smtplib.SMTP(server)
             smtp_server.sendmail(addr_from, addr_to, msg.as_string())
         except smtplib.SMTPException as err:
             smtp_server = smtplib.SMTP(server)
             smtp_server.sendmail(addr_from, addr_to, msg.as_string())
         except smtplib.SMTPException as err:
-            raise AlertingError("Not possible to send the alert via email.",
+            raise AlertingError(u"Not possible to send the alert via email.",
                                 str(err))
         finally:
             if smtp_server:
                 smtp_server.quit()
 
                                 str(err))
         finally:
             if smtp_server:
                 smtp_server.quit()
 
-    def _create_alert_message(self, alert):
-        """Create the message which is used in the generated alert.
+    def _get_compressed_failed_tests(self, alert, test_set, sort=True):
+        """Return the dictionary with compressed faild tests. The compression is
+        done by grouping the tests from the same area but with different NICs,
+        frame sizes and number of processor cores.
+
+        For example, the failed tests:
+          10ge2p1x520-64b-1c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
+          10ge2p1x520-64b-2c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
+          10ge2p1x520-64b-4c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
+          10ge2p1x520-imix-1c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
+          10ge2p1x520-imix-2c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
+          10ge2p1x520-imix-4c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
+
+        will be represented as:
+          ethip4udp-ip4scale4000-udpsrcscale15-nat44 \
+          (10ge2p1x520, 64b, imix, 1c, 2c, 4c)
+
+        Structure of returned data:
+
+        {
+            "trimmed_TC_name_1": {
+                "nics": [],
+                "framesizes": [],
+                "cores": []
+            }
+            ...
+            "trimmed_TC_name_N": {
+                "nics": [],
+                "framesizes": [],
+                "cores": []
+            }
+        }
 
 
-        :param alert: Message is created for this alert.
+        :param alert: Files are created for this alert.
+        :param test_set: Specifies which set of tests will be included in the
+            result. Its name is the same as the name of file with failed tests.
+        :param sort: If True, the failed tests are sorted alphabetically.
         :type alert: dict
         :type alert: dict
-        :returns: Message in the ASCII text and HTML format.
-        :rtype: tuple(str, str)
+        :type test_set: str
+        :type sort: bool
+        :returns: CSIT build number, VPP version, Number of passed tests,
+            Number of failed tests, Compressed failed tests.
+        :rtype: tuple(str, str, int, int, OrderedDict)
         """
 
         """
 
-        if alert["type"] == "failed-tests":
-            text = ""
-            html = "<html><body>"
-            for item in alert["include"]:
-                file_name = "{path}/{name}".format(
-                    path=self._path_failed_tests, name=item)
-                try:
-                    with open("{0}.txt".format(file_name), 'r') as txt_file:
-                        text += "{0}:\n\n".format(
-                            item.replace("failed-tests-", ""))
-                        text += txt_file.read() + "\n" * 2
-                except IOError:
-                    logging.error("Not possible to read the file '{0}.txt'.".
-                                  format(file_name))
-                try:
-                    with open("{0}.rst".format(file_name), 'r') as rst_file:
-                        html += "<h2>{0}:</h2>".format(
-                            item.replace("failed-tests-", ""))
-                        html += rst_file.readlines()[2].\
-                            replace("../trending", alert.get("url", ""))
-                        html += "<br>" * 3
-                except IOError:
-                    logging.error("Not possible to read the file '{0}.rst'.".
-                                  format(file_name))
-            html += "</body></html>"
-        else:
-            raise AlertingError("Alert of type '{0}' is not implemented.".
-                                format(alert["type"]))
-        return text, html
-
-    def _generate_files_for_jenkins(self, alert):
+        directory = self.configs[alert[u"way"]][u"output-dir"]
+        failed_tests = OrderedDict()
+        file_path = f"{directory}/{test_set}.txt"
+        version = u""
+        try:
+            with open(file_path, u'r') as f_txt:
+                for idx, line in enumerate(f_txt):
+                    if idx == 0:
+                        build = line[:-1]
+                        continue
+                    if idx == 1:
+                        version = line[:-1]
+                        continue
+                    if idx == 2:
+                        passed = line[:-1]
+                        continue
+                    if idx == 3:
+                        failed = line[:-1]
+                        continue
+                    try:
+                        test = line[:-1].split(u'-')
+                        name = u'-'.join(test[3:-1])
+                    except IndexError:
+                        continue
+                    if failed_tests.get(name, None) is None:
+                        failed_tests[name] = dict(nics=list(),
+                                                  framesizes=list(),
+                                                  cores=list())
+                    if test[0] not in failed_tests[name][u"nics"]:
+                        failed_tests[name][u"nics"].append(test[0])
+                    if test[1] not in failed_tests[name][u"framesizes"]:
+                        failed_tests[name][u"framesizes"].append(test[1])
+                    if test[2] not in failed_tests[name][u"cores"]:
+                        failed_tests[name][u"cores"].append(test[2])
+        except IOError:
+            logging.error(f"No such file or directory: {file_path}")
+            return None, None, None, None, None
+        if sort:
+            sorted_failed_tests = OrderedDict()
+            for key in sorted(failed_tests.keys()):
+                sorted_failed_tests[key] = failed_tests[key]
+            return build, version, passed, failed, sorted_failed_tests
+
+        return build, version, passed, failed, failed_tests
+
+    def _list_gressions(self, alert, idx, header, re_pro):
+        """Create a file with regressions or progressions for the test set
+        specified by idx.
+
+        :param alert: Files are created for this alert.
+        :param idx: Index of the test set as it is specified in the
+            specification file.
+        :param header: The header of the list of [re|pro]gressions.
+        :param re_pro: 'regression' or 'progression'.
+        :type alert: dict
+        :type idx: int
+        :type header: str
+        :type re_pro: str
+        """
+
+        if re_pro not in (u"regressions", u"progressions"):
+            return
+
+        in_file = (
+            f"{self.configs[alert[u'way']][u'output-dir']}/"
+            f"{re_pro}-{alert[u'urls'][idx].split(u'/')[-1]}.txt"
+        )
+        out_file = (
+            f"{self.configs[alert[u'way']][u'output-dir']}/"
+            f"trending-{re_pro}.txt"
+        )
+
+        try:
+            with open(in_file, u'r') as txt_file:
+                file_content = txt_file.read()
+                with open(out_file, u'a+') as reg_file:
+                    reg_file.write(header)
+                    if file_content:
+                        reg_file.write(file_content)
+                    else:
+                        reg_file.write(f"No {re_pro}")
+        except IOError as err:
+            logging.warning(repr(err))
+
+    def _generate_email_body(self, alert):
         """Create the file which is used in the generated alert.
 
         :param alert: Files are created for this alert.
         :type alert: dict
         """
 
         """Create the file which is used in the generated alert.
 
         :param alert: Files are created for this alert.
         :type alert: dict
         """
 
-        config = self.configs[alert["way"]]
-
-        if alert["type"] == "failed-tests":
-            text, html = self._create_alert_message(alert)
-            file_name = "{0}/{1}".format(config["output-dir"],
-                                         config["output-file"])
-            logging.info("Writing the file '{0}.txt' ...".format(file_name))
-            try:
-                with open("{0}.txt".format(file_name), 'w') as txt_file:
-                    txt_file.write(text)
-            except IOError:
-                logging.error("Not possible to write the file '{0}.txt'.".
-                              format(file_name))
-            logging.info("Writing the file '{0}.html' ...".format(file_name))
-            try:
-                with open("{0}.html".format(file_name), 'w') as html_file:
-                    html_file.write(html)
-            except IOError:
-                logging.error("Not possible to write the file '{0}.html'.".
-                              format(file_name))
-
-            zip_file = config.get("zip-output", None)
-            if zip_file:
-                logging.info("Writing the file '{0}/{1}' ...".
-                             format(config["output-dir"], zip_file))
-                execute_command("tar czvf {dir}/{zip} --directory={dir} "
-                                "{input}.txt {input}.html".
-                                format(dir=config["output-dir"],
-                                       zip=zip_file,
-                                       input=config["output-file"]))
-        else:
-            raise AlertingError("Alert of type '{0}' is not implemented.".
-                                format(alert["type"]))
+        if alert[u"type"] != u"failed-tests":
+            raise AlertingError(
+                f"Alert of type {alert[u'type']} is not implemented."
+            )
+
+        text = u""
+        for idx, test_set in enumerate(alert.get(u"include", [])):
+            build, version, passed, failed, failed_tests = \
+                self._get_compressed_failed_tests(alert, test_set)
+            if build is None:
+                ret_code, build_nr, _ = get_last_completed_build_number(
+                    self._spec.environment[u"urls"][u"URL[JENKINS,CSIT]"],
+                    alert[u"urls"][idx].split(u'/')[-1])
+                if ret_code != 0:
+                    build_nr = u''
+                text += (
+                    f"\n\nNo input data available for "
+                    f"{u'-'.join(test_set.split('-')[-2:])}. See CSIT build "
+                    f"{alert[u'urls'][idx]}/{build_nr} for more information.\n"
+                )
+                continue
+            text += (
+                f"\n\n{test_set.split('-')[-2]}-{test_set.split('-')[-1]}, "
+                f"{failed} tests failed, "
+                f"{passed} tests passed, CSIT build: "
+                f"{alert[u'urls'][idx]}/{build}, VPP version: {version}\n\n"
+            )
+
+            class MaxLens():
+                """Class to store the max lengths of strings displayed in
+                failed tests list.
+                """
+                def __init__(self, tst_name, nics, framesizes, cores):
+                    """Initialisation.
+
+                    :param tst_name: Name of the test.
+                    :param nics: NICs used in the test.
+                    :param framesizes: Frame sizes used in the tests
+                    :param cores: Cores used in th test.
+                    """
+                    self.name = tst_name
+                    self.nics = nics
+                    self.frmsizes = framesizes
+                    self.cores = cores
+
+            max_len = MaxLens(0, 0, 0, 0)
+
+            for name, params in failed_tests.items():
+                failed_tests[name][u"nics"] = u",".join(sorted(params[u"nics"]))
+                failed_tests[name][u"framesizes"] = \
+                    u",".join(sorted(params[u"framesizes"]))
+                failed_tests[name][u"cores"] = \
+                    u",".join(sorted(params[u"cores"]))
+                if len(name) > max_len.name:
+                    max_len.name = len(name)
+                if len(failed_tests[name][u"nics"]) > max_len.nics:
+                    max_len.nics = len(failed_tests[name][u"nics"])
+                if len(failed_tests[name][u"framesizes"]) > max_len.frmsizes:
+                    max_len.frmsizes = len(failed_tests[name][u"framesizes"])
+                if len(failed_tests[name][u"cores"]) > max_len.cores:
+                    max_len.cores = len(failed_tests[name][u"cores"])
+
+            for name, params in failed_tests.items():
+                text += (
+                    f"{name + u' ' * (max_len.name - len(name))}  "
+                    f"{params[u'nics']}"
+                    f"{u' ' * (max_len.nics - len(params[u'nics']))}  "
+                    f"{params[u'framesizes']}"
+                    f"{u' ' * (max_len.frmsizes-len(params[u'framesizes']))}  "
+                    f"{params[u'cores']}"
+                    f"{u' ' * (max_len.cores - len(params[u'cores']))}\n"
+                )
+
+            gression_hdr = (
+                f"\n\n{test_set.split(u'-')[-2]}-{test_set.split(u'-')[-1]}, "
+                f"CSIT build: {alert[u'urls'][idx]}/{build}, "
+                f"VPP version: {version}\n\n"
+            )
+            # Add list of regressions:
+            self._list_gressions(alert, idx, gression_hdr, u"regressions")
+
+            # Add list of progressions:
+            self._list_gressions(alert, idx, gression_hdr, u"progressions")
+
+        text += f"\nFor detailed information visit: {alert[u'url-details']}\n"
+        file_name = f"{self.configs[alert[u'way']][u'output-dir']}/" \
+                    f"{self.configs[alert[u'way']][u'output-file']}"
+        logging.info(f"Writing the file {file_name}.txt ...")
+
+        try:
+            with open(f"{file_name}.txt", u'w') as txt_file:
+                txt_file.write(text)
+        except IOError:
+            logging.error(f"Not possible to write the file {file_name}.txt.")