resources/tools/presentation/generator_alerts.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Generator of alerts:
  15 - failed tests
  16 - regressions
  17 - progressions
  18 """
  19
  20
  21 import smtplib
  22 import logging
  23 import re
  24
  25 from email.mime.text import MIMEText
  26 from email.mime.multipart import MIMEMultipart
  27 from os.path import isdir
  28 from collections import OrderedDict
  29
  30 from pal_utils import get_last_completed_build_number
  31 from pal_errors import PresentationError
  32
  33
  34 class AlertingError(PresentationError):
  35     """Exception(s) raised by the alerting module.
  36
  37     When raising this exception, put this information to the message in this
  38     order:
  39      - short description of the encountered problem (parameter msg),
  40      - relevant messages if there are any collected, e.g., from caught
  41        exception (optional parameter details),
  42      - relevant data if there are any collected (optional parameter details).
  43     """
  44
  45     def __init__(self, msg, details=u'', level=u"CRITICAL"):
  46         """Sets the exception message and the level.
  47
  48         :param msg: Short description of the encountered problem.
  49         :param details: Relevant messages if there are any collected, e.g.,
  50             from caught exception (optional parameter details), or relevant data
  51             if there are any collected (optional parameter details).
  52         :param level: Level of the error, possible choices are: "DEBUG", "INFO",
  53             "WARNING", "ERROR" and "CRITICAL".
  54         :type msg: str
  55         :type details: str
  56         :type level: str
  57         """
  58
  59         super(AlertingError, self).__init__(f"Alerting: {msg}", details, level)
  60
  61     def __repr__(self):
  62         return (
  63             f"AlertingError(msg={self._msg!r},details={self._details!r},"
  64             f"level={self._level!r})"
  65         )
  66
  67
  68 class Alerting:
  69     """Class implementing the alerting mechanism.
  70     """
  71
  72     def __init__(self, spec):
  73         """Initialization.
  74
  75         :param spec: The CPTA specification.
  76         :type spec: Specification
  77         """
  78
  79         # Implemented alerts:
  80         self._implemented_alerts = (u"failed-tests", )
  81
  82         self._spec = spec
  83
  84         try:
  85             self._spec_alert = spec.alerting
  86         except KeyError as err:
  87             raise AlertingError(u"Alerting is not configured, skipped.",
  88                                 repr(err),
  89                                 u"WARNING")
  90
  91         self._path_failed_tests = spec.environment[u"paths"][u"DIR[STATIC,VPP]"]
  92
  93         # Verify and validate input specification:
  94         self.configs = self._spec_alert.get(u"configurations", None)
  95         if not self.configs:
  96             raise AlertingError(u"No alert configuration is specified.")
  97         for config_type, config_data in self.configs.items():
  98             if config_type == u"email":
  99                 if not config_data.get(u"server", None):
 100                     raise AlertingError(u"Parameter 'server' is missing.")
 101                 if not config_data.get(u"address-to", None):
 102                     raise AlertingError(u"Parameter 'address-to' (recipient) "
 103                                         u"is missing.")
 104                 if not config_data.get(u"address-from", None):
 105                     raise AlertingError(u"Parameter 'address-from' (sender) is "
 106                                         u"missing.")
 107             elif config_type == u"jenkins":
 108                 if not isdir(config_data.get(u"output-dir", u"")):
 109                     raise AlertingError(u"Parameter 'output-dir' is "
 110                                         u"missing or it is not a directory.")
 111                 if not config_data.get(u"output-file", None):
 112                     raise AlertingError(u"Parameter 'output-file' is missing.")
 113             else:
 114                 raise AlertingError(
 115                     f"Alert of type {config_type} is not implemented."
 116                 )
 117
 118         self.alerts = self._spec_alert.get(u"alerts", None)
 119         if not self.alerts:
 120             raise AlertingError(u"No alert is specified.")
 121         for alert_data in self.alerts.values():
 122             if not alert_data.get(u"title", None):
 123                 raise AlertingError(u"Parameter 'title' is missing.")
 124             if not alert_data.get(u"type", None) in self._implemented_alerts:
 125                 raise AlertingError(u"Parameter 'failed-tests' is missing or "
 126                                     u"incorrect.")
 127             if not alert_data.get(u"way", None) in self.configs.keys():
 128                 raise AlertingError(u"Parameter 'way' is missing or incorrect.")
 129             if not alert_data.get(u"include", None):
 130                 raise AlertingError(u"Parameter 'include' is missing or the "
 131                                     u"list is empty.")
 132
 133     def __str__(self):
 134         """Return string with human readable description of the alert.
 135
 136         :returns: Readable description.
 137         :rtype: str
 138         """
 139         return f"configs={self.configs}, alerts={self.alerts}"
 140
 141     def __repr__(self):
 142         """Return string executable as Python constructor call.
 143
 144         :returns: Executable constructor call.
 145         :rtype: str
 146         """
 147         return f"Alerting(spec={self._spec})"
 148
 149     def generate_alerts(self):
 150         """Generate alert(s) using specified way(s).
 151         """
 152
 153         for alert_data in self.alerts.values():
 154             if alert_data[u"way"] == u"jenkins":
 155                 self._generate_email_body(alert_data)
 156             else:
 157                 raise AlertingError(
 158                     f"Alert with way {alert_data[u'way']} is not implemented."
 159                 )
 160
 161     @staticmethod
 162     def _send_email(server, addr_from, addr_to, subject, text=None, html=None):
 163         """Send an email using predefined configuration.
 164
 165         :param server: SMTP server used to send email.
 166         :param addr_from: Sender address.
 167         :param addr_to: Recipient address(es).
 168         :param subject: Subject of the email.
 169         :param text: Message in the ASCII text format.
 170         :param html: Message in the HTML format.
 171         :type server: str
 172         :type addr_from: str
 173         :type addr_to: list
 174         :type subject: str
 175         :type text: str
 176         :type html: str
 177         """
 178
 179         if not text and not html:
 180             raise AlertingError(u"No text/data to send.")
 181
 182         msg = MIMEMultipart(u'alternative')
 183         msg[u'Subject'] = subject
 184         msg[u'From'] = addr_from
 185         msg[u'To'] = u", ".join(addr_to)
 186
 187         if text:
 188             msg.attach(MIMEText(text, u'plain'))
 189         if html:
 190             msg.attach(MIMEText(html, u'html'))
 191
 192         smtp_server = None
 193         try:
 194             logging.info(f"Trying to send alert {subject} ...")
 195             logging.debug(f"SMTP Server: {server}")
 196             logging.debug(f"From: {addr_from}")
 197             logging.debug(f"To: {u', '.join(addr_to)}")
 198             logging.debug(f"Message: {msg.as_string()}")
 199             smtp_server = smtplib.SMTP(server)
 200             smtp_server.sendmail(addr_from, addr_to, msg.as_string())
 201         except smtplib.SMTPException as err:
 202             raise AlertingError(u"Not possible to send the alert via email.",
 203                                 str(err))
 204         finally:
 205             if smtp_server:
 206                 smtp_server.quit()
 207
 208     def _get_compressed_failed_tests(self, alert, test_set, sort=True):
 209         """Return the dictionary with compressed faild tests. The compression is
 210         done by grouping the tests from the same area but with different NICs,
 211         frame sizes and number of processor cores.
 212
 213         For example, the failed tests:
 214           10ge2p1x520-64b-1c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
 215           10ge2p1x520-64b-2c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
 216           10ge2p1x520-64b-4c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
 217           10ge2p1x520-imix-1c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
 218           10ge2p1x520-imix-2c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
 219           10ge2p1x520-imix-4c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
 220
 221         will be represented as:
 222           ethip4udp-ip4scale4000-udpsrcscale15-nat44 \
 223           (10ge2p1x520, 64b, imix, 1c, 2c, 4c)
 224
 225         Structure of returned data:
 226
 227         {
 228             "trimmed_TC_name_1": {
 229                 "nics": [],
 230                 "framesizes": [],
 231                 "cores": []
 232             }
 233             ...
 234             "trimmed_TC_name_N": {
 235                 "nics": [],
 236                 "framesizes": [],
 237                 "cores": []
 238             }
 239         }
 240
 241         :param alert: Files are created for this alert.
 242         :param test_set: Specifies which set of tests will be included in the
 243             result. Its name is the same as the name of file with failed tests.
 244         :param sort: If True, the failed tests are sorted alphabetically.
 245         :type alert: dict
 246         :type test_set: str
 247         :type sort: bool
 248         :returns: CSIT build number, VPP version, Number of passed tests,
 249             Number of failed tests, Compressed failed tests.
 250         :rtype: tuple(str, str, int, int, OrderedDict)
 251         """
 252
 253         directory = self.configs[alert[u"way"]][u"output-dir"]
 254         failed_tests = OrderedDict()
 255         file_path = f"{directory}/{test_set}.txt"
 256         version = u""
 257         try:
 258             with open(file_path, u'r') as f_txt:
 259                 for idx, line in enumerate(f_txt):
 260                     if idx == 0:
 261                         build = line[:-1]
 262                         continue
 263                     if idx == 1:
 264                         version = line[:-1]
 265                         continue
 266                     if idx == 2:
 267                         passed = line[:-1]
 268                         continue
 269                     if idx == 3:
 270                         failed = line[:-1]
 271                         continue
 272                     try:
 273                         test = line[:-1].split(u'-')
 274                         name = u'-'.join(test[3:-1])
 275                     except IndexError:
 276                         continue
 277                     if failed_tests.get(name, None) is None:
 278                         failed_tests[name] = dict(nics=list(),
 279                                                   framesizes=list(),
 280                                                   cores=list())
 281                     if test[0] not in failed_tests[name][u"nics"]:
 282                         failed_tests[name][u"nics"].append(test[0])
 283                     if test[1] not in failed_tests[name][u"framesizes"]:
 284                         failed_tests[name][u"framesizes"].append(test[1])
 285                     if test[2] not in failed_tests[name][u"cores"]:
 286                         failed_tests[name][u"cores"].append(test[2])
 287         except IOError:
 288             logging.error(f"No such file or directory: {file_path}")
 289             return None, None, None, None, None
 290         if sort:
 291             sorted_failed_tests = OrderedDict()
 292             for key in sorted(failed_tests.keys()):
 293                 sorted_failed_tests[key] = failed_tests[key]
 294             return build, version, passed, failed, sorted_failed_tests
 295
 296         return build, version, passed, failed, failed_tests
 297
 298     def _list_gressions(self, alert, idx, header, re_pro):
 299         """Create a file with regressions or progressions for the test set
 300         specified by idx.
 301
 302         :param alert: Files are created for this alert.
 303         :param idx: Index of the test set as it is specified in the
 304             specification file.
 305         :param header: The header of the list of [re|pro]gressions.
 306         :param re_pro: 'regression' or 'progression'.
 307         :type alert: dict
 308         :type idx: int
 309         :type header: str
 310         :type re_pro: str
 311         """
 312
 313         if re_pro not in (u"regressions", u"progressions"):
 314             return
 315
 316         in_file = (
 317             f"{self.configs[alert[u'way']][u'output-dir']}/"
 318             f"{re_pro}-{alert[u'urls'][idx].split(u'/')[-1]}.txt"
 319         )
 320         out_file = (
 321             f"{self.configs[alert[u'way']][u'output-dir']}/"
 322             f"trending-{re_pro}.txt"
 323         )
 324
 325         try:
 326             with open(in_file, u'r') as txt_file:
 327                 file_content = txt_file.read()
 328                 with open(out_file, u'a+') as reg_file:
 329                     reg_file.write(header)
 330                     if file_content:
 331                         reg_file.write(file_content)
 332                     else:
 333                         reg_file.write(f"No {re_pro}")
 334         except IOError as err:
 335             logging.warning(repr(err))
 336
 337     def _generate_email_body(self, alert):
 338         """Create the file which is used in the generated alert.
 339
 340         :param alert: Files are created for this alert.
 341         :type alert: dict
 342         """
 343
 344         if alert[u"type"] != u"failed-tests":
 345             raise AlertingError(
 346                 f"Alert of type {alert[u'type']} is not implemented."
 347             )
 348
 349         text = u""
 350         for idx, test_set in enumerate(alert.get(u"include", list())):
 351             test_set_short = u""
 352             device = u""
 353             try:
 354                 groups = re.search(
 355                     re.compile(r'((vpp|dpdk)-\dn-(skx|clx|hsw|tsh|dnv)-.*)'),
 356                     test_set
 357                 )
 358                 test_set_short = groups.group(1)
 359                 device = groups.group(2)
 360             except (AttributeError, IndexError):
 361                 logging.error(
 362                     f"The test set {test_set} does not include information "
 363                     f"about test bed. Using empty string instead."
 364                 )
 365             build, version, passed, failed, failed_tests = \
 366                 self._get_compressed_failed_tests(alert, test_set)
 367             if build is None:
 368                 ret_code, build_nr, _ = get_last_completed_build_number(
 369                     self._spec.environment[u"urls"][u"URL[JENKINS,CSIT]"],
 370                     alert[u"urls"][idx].split(u'/')[-1])
 371                 if ret_code != 0:
 372                     build_nr = u''
 373                 text += (
 374                     f"\n\nNo input data available for {test_set_short}. "
 375                     f"See CSIT build {alert[u'urls'][idx]}/{build_nr} for more "
 376                     f"information.\n"
 377                 )
 378                 continue
 379             text += (
 380                 f"\n\n{test_set_short}, {failed} tests failed, {passed} tests "
 381                 f"passed, CSIT build: {alert[u'urls'][idx]}/{build}, "
 382                 f"{device} version: {version}\n\n"
 383             )
 384
 385             class MaxLens():
 386                 """Class to store the max lengths of strings displayed in
 387                 failed tests list.
 388                 """
 389                 def __init__(self, tst_name, nics, framesizes, cores):
 390                     """Initialisation.
 391
 392                     :param tst_name: Name of the test.
 393                     :param nics: NICs used in the test.
 394                     :param framesizes: Frame sizes used in the tests
 395                     :param cores: Cores used in th test.
 396                     """
 397                     self.name = tst_name
 398                     self.nics = nics
 399                     self.frmsizes = framesizes
 400                     self.cores = cores
 401
 402             max_len = MaxLens(0, 0, 0, 0)
 403
 404             for name, params in failed_tests.items():
 405                 failed_tests[name][u"nics"] = u",".join(sorted(params[u"nics"]))
 406                 failed_tests[name][u"framesizes"] = \
 407                     u",".join(sorted(params[u"framesizes"]))
 408                 failed_tests[name][u"cores"] = \
 409                     u",".join(sorted(params[u"cores"]))
 410                 if len(name) > max_len.name:
 411                     max_len.name = len(name)
 412                 if len(failed_tests[name][u"nics"]) > max_len.nics:
 413                     max_len.nics = len(failed_tests[name][u"nics"])
 414                 if len(failed_tests[name][u"framesizes"]) > max_len.frmsizes:
 415                     max_len.frmsizes = len(failed_tests[name][u"framesizes"])
 416                 if len(failed_tests[name][u"cores"]) > max_len.cores:
 417                     max_len.cores = len(failed_tests[name][u"cores"])
 418
 419             for name, params in failed_tests.items():
 420                 text += (
 421                     f"{name + u' ' * (max_len.name - len(name))}  "
 422                     f"{params[u'nics']}"
 423                     f"{u' ' * (max_len.nics - len(params[u'nics']))}  "
 424                     f"{params[u'framesizes']}"
 425                     f"{u' ' * (max_len.frmsizes-len(params[u'framesizes']))}  "
 426                     f"{params[u'cores']}"
 427                     f"{u' ' * (max_len.cores - len(params[u'cores']))}\n"
 428                 )
 429
 430             gression_hdr = (
 431                 f"\n\n{test_set_short}, "
 432                 f"CSIT build: {alert[u'urls'][idx]}/{build}, "
 433                 f"{device} version: {version}\n\n"
 434             )
 435             # Add list of regressions:
 436             self._list_gressions(alert, idx, gression_hdr, u"regressions")
 437
 438             # Add list of progressions:
 439             self._list_gressions(alert, idx, gression_hdr, u"progressions")
 440
 441         text += f"\nFor detailed information visit: {alert[u'url-details']}\n"
 442         file_name = f"{self.configs[alert[u'way']][u'output-dir']}/" \
 443                     f"{self.configs[alert[u'way']][u'output-file']}"
 444         logging.info(f"Writing the file {file_name}.txt ...")
 445
 446         try:
 447             with open(f"{file_name}.txt", u'w') as txt_file:
 448                 txt_file.write(text)
 449         except IOError:
 450             logging.error(f"Not possible to write the file {file_name}.txt.")