resources/tools/presentation/generator_alerts.py

   1 # Copyright (c) 2021 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Generator of alerts:
  15 - failed tests
  16 - regressions
  17 - progressions
  18 """
  19
  20
  21 import smtplib
  22 import logging
  23 import re
  24
  25 from difflib import SequenceMatcher
  26 from email.mime.text import MIMEText
  27 from email.mime.multipart import MIMEMultipart
  28 from os.path import isdir
  29 from collections import OrderedDict, defaultdict
  30
  31 from pal_errors import PresentationError
  32
  33
  34 class AlertingError(PresentationError):
  35     """Exception(s) raised by the alerting module.
  36
  37     When raising this exception, put this information to the message in this
  38     order:
  39      - short description of the encountered problem (parameter msg),
  40      - relevant messages if there are any collected, e.g., from caught
  41        exception (optional parameter details),
  42      - relevant data if there are any collected (optional parameter details).
  43     """
  44
  45     def __init__(self, msg, details=u'', level=u"CRITICAL"):
  46         """Sets the exception message and the level.
  47
  48         :param msg: Short description of the encountered problem.
  49         :param details: Relevant messages if there are any collected, e.g.,
  50             from caught exception (optional parameter details), or relevant data
  51             if there are any collected (optional parameter details).
  52         :param level: Level of the error, possible choices are: "DEBUG", "INFO",
  53             "WARNING", "ERROR" and "CRITICAL".
  54         :type msg: str
  55         :type details: str
  56         :type level: str
  57         """
  58
  59         super(AlertingError, self).__init__(f"Alerting: {msg}", details, level)
  60
  61     def __repr__(self):
  62         return (
  63             f"AlertingError(msg={self._msg!r},details={self._details!r},"
  64             f"level={self._level!r})"
  65         )
  66
  67
  68 class Alerting:
  69     """Class implementing the alerting mechanism.
  70     """
  71
  72     def __init__(self, spec):
  73         """Initialization.
  74
  75         :param spec: The CPTA specification.
  76         :type spec: Specification
  77         """
  78
  79         # Implemented alerts:
  80         self._implemented_alerts = (u"failed-tests", )
  81
  82         self._spec = spec
  83
  84         self.error_msgs = list()
  85
  86         try:
  87             self._spec_alert = spec.alerting
  88         except KeyError as err:
  89             raise AlertingError(
  90                 u"Alerting is not configured, skipped.", repr(err), u"WARNING"
  91             )
  92
  93         self._path_failed_tests = spec.environment[u"paths"][u"DIR[STATIC,VPP]"]
  94
  95         # Verify and validate input specification:
  96         self.configs = self._spec_alert.get(u"configurations", None)
  97         if not self.configs:
  98             raise AlertingError(u"No alert configuration is specified.")
  99         for config_type, config_data in self.configs.items():
 100             if config_type == u"email":
 101                 if not config_data.get(u"server", None):
 102                     raise AlertingError(u"Parameter 'server' is missing.")
 103                 if not config_data.get(u"address-to", None):
 104                     raise AlertingError(u"Parameter 'address-to' (recipient) "
 105                                         u"is missing.")
 106                 if not config_data.get(u"address-from", None):
 107                     raise AlertingError(u"Parameter 'address-from' (sender) is "
 108                                         u"missing.")
 109             elif config_type == u"jenkins":
 110                 if not isdir(config_data.get(u"output-dir", u"")):
 111                     raise AlertingError(u"Parameter 'output-dir' is "
 112                                         u"missing or it is not a directory.")
 113                 if not config_data.get(u"output-file", None):
 114                     raise AlertingError(u"Parameter 'output-file' is missing.")
 115             else:
 116                 raise AlertingError(
 117                     f"Alert of type {config_type} is not implemented."
 118                 )
 119
 120         self.alerts = self._spec_alert.get(u"alerts", None)
 121         if not self.alerts:
 122             raise AlertingError(u"No alert is specified.")
 123         for alert_data in self.alerts.values():
 124             if not alert_data.get(u"title", None):
 125                 raise AlertingError(u"Parameter 'title' is missing.")
 126             if not alert_data.get(u"type", None) in self._implemented_alerts:
 127                 raise AlertingError(u"Parameter 'failed-tests' is missing or "
 128                                     u"incorrect.")
 129             if not alert_data.get(u"way", None) in self.configs.keys():
 130                 raise AlertingError(u"Parameter 'way' is missing or incorrect.")
 131             if not alert_data.get(u"include", None):
 132                 raise AlertingError(u"Parameter 'include' is missing or the "
 133                                     u"list is empty.")
 134
 135     def __str__(self):
 136         """Return string with human readable description of the alert.
 137
 138         :returns: Readable description.
 139         :rtype: str
 140         """
 141         return f"configs={self.configs}, alerts={self.alerts}"
 142
 143     def __repr__(self):
 144         """Return string executable as Python constructor call.
 145
 146         :returns: Executable constructor call.
 147         :rtype: str
 148         """
 149         return f"Alerting(spec={self._spec})"
 150
 151     def generate_alerts(self):
 152         """Generate alert(s) using specified way(s).
 153         """
 154
 155         for alert_data in self.alerts.values():
 156             if alert_data[u"way"] == u"jenkins":
 157                 self._generate_email_body(alert_data)
 158             else:
 159                 raise AlertingError(
 160                     f"Alert with way {alert_data[u'way']} is not implemented."
 161                 )
 162
 163     @staticmethod
 164     def _send_email(server, addr_from, addr_to, subject, text=None, html=None):
 165         """Send an email using predefined configuration.
 166
 167         :param server: SMTP server used to send email.
 168         :param addr_from: Sender address.
 169         :param addr_to: Recipient address(es).
 170         :param subject: Subject of the email.
 171         :param text: Message in the ASCII text format.
 172         :param html: Message in the HTML format.
 173         :type server: str
 174         :type addr_from: str
 175         :type addr_to: list
 176         :type subject: str
 177         :type text: str
 178         :type html: str
 179         """
 180
 181         if not text and not html:
 182             raise AlertingError(u"No text/data to send.")
 183
 184         msg = MIMEMultipart(u'alternative')
 185         msg[u'Subject'] = subject
 186         msg[u'From'] = addr_from
 187         msg[u'To'] = u", ".join(addr_to)
 188
 189         if text:
 190             msg.attach(MIMEText(text, u'plain'))
 191         if html:
 192             msg.attach(MIMEText(html, u'html'))
 193
 194         smtp_server = None
 195         try:
 196             logging.info(f"Trying to send alert {subject} ...")
 197             logging.debug(f"SMTP Server: {server}")
 198             logging.debug(f"From: {addr_from}")
 199             logging.debug(f"To: {u', '.join(addr_to)}")
 200             logging.debug(f"Message: {msg.as_string()}")
 201             smtp_server = smtplib.SMTP(server)
 202             smtp_server.sendmail(addr_from, addr_to, msg.as_string())
 203         except smtplib.SMTPException as err:
 204             raise AlertingError(u"Not possible to send the alert via email.",
 205                                 str(err))
 206         finally:
 207             if smtp_server:
 208                 smtp_server.quit()
 209
 210     def _get_compressed_failed_tests(self, alert, test_set, sort=True):
 211         """Return the dictionary with compressed faild tests. The compression is
 212         done by grouping the tests from the same area but with different NICs,
 213         frame sizes and number of processor cores.
 214
 215         For example, the failed tests:
 216           10ge2p1x520-64b-1c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
 217           10ge2p1x520-64b-2c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
 218           10ge2p1x520-64b-4c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
 219           10ge2p1x520-imix-1c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
 220           10ge2p1x520-imix-2c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
 221           10ge2p1x520-imix-4c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
 222
 223         will be represented as:
 224           ethip4udp-ip4scale4000-udpsrcscale15-nat44 \
 225           (10ge2p1x520, 64b, imix, 1c, 2c, 4c)
 226
 227         Structure of returned data:
 228
 229         {
 230             "trimmed_TC_name_1": {
 231                 "nics": [],
 232                 "framesizes": [],
 233                 "cores": []
 234             }
 235             ...
 236             "trimmed_TC_name_N": {
 237                 "nics": [],
 238                 "framesizes": [],
 239                 "cores": []
 240             }
 241         }
 242
 243         :param alert: Files are created for this alert.
 244         :param test_set: Specifies which set of tests will be included in the
 245             result. Its name is the same as the name of file with failed tests.
 246         :param sort: If True, the failed tests are sorted alphabetically.
 247         :type alert: dict
 248         :type test_set: str
 249         :type sort: bool
 250         :returns: CSIT build number, VPP version, Number of passed tests,
 251             Number of failed tests, Compressed failed tests.
 252         :rtype: tuple(str, str, int, int, str, OrderedDict)
 253         """
 254
 255         directory = self.configs[alert[u"way"]][u"output-dir"]
 256         failed_tests = defaultdict(dict)
 257         file_path = f"{directory}/{test_set}.txt"
 258         version = u""
 259         try:
 260             with open(file_path, u'r') as f_txt:
 261                 for idx, line in enumerate(f_txt):
 262                     if idx == 0:
 263                         build = line[:-1]
 264                         continue
 265                     if idx == 1:
 266                         version = line[:-1]
 267                         continue
 268                     if idx == 2:
 269                         passed = line[:-1]
 270                         continue
 271                     if idx == 3:
 272                         failed = line[:-1]
 273                         continue
 274                     if idx == 4:
 275                         minutes = int(line[:-1]) // 60000
 276                         duration = f"{(minutes // 60):02d}:{(minutes % 60):02d}"
 277                         continue
 278                     try:
 279                         line, error_msg = line[:-1].split(u'###', maxsplit=1)
 280                         test = line.split(u'-')
 281                         name = u'-'.join(test[3:-1])
 282                     except ValueError:
 283                         continue
 284
 285                     for e_msg in self.error_msgs:
 286                         if SequenceMatcher(None, e_msg,
 287                                            error_msg).ratio() > 0.5:
 288                             error_msg = e_msg
 289                             break
 290                     if error_msg not in self.error_msgs:
 291                         self.error_msgs.append(error_msg)
 292
 293                     error_msg_index = self.error_msgs.index(error_msg)
 294
 295                     if failed_tests.get(name, {}).get(error_msg_index) is None:
 296                         failed_tests[name][error_msg_index] = \
 297                             dict(nics=list(),
 298                                  framesizes=list(),
 299                                  cores=list())
 300
 301                     if test[0] not in \
 302                             failed_tests[name][error_msg_index][u"nics"]:
 303                         failed_tests[name][error_msg_index][u"nics"].\
 304                             append(test[0])
 305                     if test[1] not in \
 306                             failed_tests[name][error_msg_index][u"framesizes"]:
 307                         failed_tests[name][error_msg_index][u"framesizes"].\
 308                             append(test[1])
 309                     check_core = test[2] + f"[{str(error_msg_index)}]"
 310                     if check_core not in \
 311                             failed_tests[name][error_msg_index][u"cores"]:
 312                         failed_tests[name][error_msg_index][u"cores"].\
 313                             append(test[2] + "[" + str(error_msg_index) + "]")
 314
 315         except IOError:
 316             logging.error(f"No such file or directory: {file_path}")
 317             return None, None, None, None, None, None
 318         if sort:
 319             sorted_failed_tests = OrderedDict()
 320             for key in sorted(failed_tests.keys()):
 321                 sorted_failed_tests[key] = failed_tests[key]
 322             return build, version, passed, failed, duration, sorted_failed_tests
 323
 324         return build, version, passed, failed, duration, failed_tests
 325
 326     def _list_gressions(self, alert, idx, header, re_pro):
 327         """Create a file with regressions or progressions for the test set
 328         specified by idx.
 329
 330         :param alert: Files are created for this alert.
 331         :param idx: Index of the test set as it is specified in the
 332             specification file.
 333         :param header: The header of the list of [re|pro]gressions.
 334         :param re_pro: 'regression' or 'progression'.
 335         :type alert: dict
 336         :type idx: int
 337         :type header: str
 338         :type re_pro: str
 339         """
 340
 341         if re_pro not in (u"regressions", u"progressions"):
 342             return
 343
 344         in_file = (
 345             f"{self.configs[alert[u'way']][u'output-dir']}/"
 346             f"{re_pro}-{alert[u'urls'][idx].split(u'/')[-1]}.txt"
 347         )
 348         out_file = (
 349             f"{self.configs[alert[u'way']][u'output-dir']}/"
 350             f"trending-{re_pro}.txt"
 351         )
 352
 353         try:
 354             with open(in_file, u'r') as txt_file:
 355                 file_content = txt_file.read()
 356                 with open(out_file, u'a+') as reg_file:
 357                     reg_file.write(header)
 358                     if file_content:
 359                         reg_file.write(file_content)
 360                     else:
 361                         reg_file.write(f"No {re_pro}")
 362         except IOError as err:
 363             logging.warning(repr(err))
 364
 365     def _generate_email_body(self, alert):
 366         """Create the file which is used in the generated alert.
 367
 368         :param alert: Files are created for this alert.
 369         :type alert: dict
 370         """
 371
 372         if alert[u"type"] != u"failed-tests":
 373             raise AlertingError(
 374                 f"Alert of type {alert[u'type']} is not implemented."
 375             )
 376
 377         text = u""
 378         for idx, test_set in enumerate(alert.get(u"include", list())):
 379             test_set_short = u""
 380             device = u""
 381             try:
 382                 groups = re.search(
 383                     re.compile(
 384                         r'((vpp|dpdk)-\dn-(skx|clx|tsh|dnv|zn2|tx2)-.*)'
 385                     ),
 386                     test_set
 387                 )
 388                 test_set_short = groups.group(1)
 389                 device = groups.group(2)
 390             except (AttributeError, IndexError):
 391                 logging.error(
 392                     f"The test set {test_set} does not include information "
 393                     f"about test bed. Using empty string instead."
 394                 )
 395             build, version, passed, failed, duration, failed_tests = \
 396                 self._get_compressed_failed_tests(alert, test_set)
 397             if build is None:
 398                 text += (
 399                     f"\n\nNo input data available for {test_set_short}. "
 400                     f"See CSIT job {alert[u'urls'][idx]} for more "
 401                     f"information.\n"
 402                 )
 403                 continue
 404             text += (
 405                 f"\n\n{test_set_short}, "
 406                 f"{failed} tests failed, "
 407                 f"{passed} tests passed, "
 408                 f"duration: {duration}, "
 409                 f"CSIT build: {alert[u'urls'][idx]}/{build}, "
 410                 f"{device} version: {version}\n\n"
 411             )
 412
 413             class MaxLens():
 414                 """Class to store the max lengths of strings displayed in
 415                 failed tests list.
 416                 """
 417                 def __init__(self, tst_name, nics, framesizes, cores):
 418                     """Initialisation.
 419
 420                     :param tst_name: Name of the test.
 421                     :param nics: NICs used in the test.
 422                     :param framesizes: Frame sizes used in the tests
 423                     :param cores: Cores used in th test.
 424                     """
 425                     self.name = tst_name
 426                     self.nics = nics
 427                     self.frmsizes = framesizes
 428                     self.cores = cores
 429
 430             max_len = MaxLens(0, 0, 0, 0)
 431
 432             for test, message in failed_tests.items():
 433                 for e_message, params in message.items():
 434                     failed_tests[test][e_message][u"nics"] = \
 435                         u" ".join(sorted(params[u"nics"]))
 436                     failed_tests[test][e_message][u"framesizes"] = \
 437                         u" ".join(sorted(params[u"framesizes"]))
 438                     failed_tests[test][e_message][u"cores"] = \
 439                         u" ".join(sorted(params[u"cores"]))
 440                     if len(test) > max_len.name:
 441                         max_len.name = len(test)
 442                     if len(failed_tests[test][e_message][u"nics"]) > \
 443                             max_len.nics:
 444                         max_len.nics = \
 445                             len(failed_tests[test][e_message][u"nics"])
 446                     if len(failed_tests[test][e_message][u"framesizes"]) > \
 447                             max_len.frmsizes:
 448                         max_len.frmsizes = \
 449                             len(failed_tests[test][e_message][u"framesizes"])
 450                     if len(failed_tests[test][e_message][u"cores"]) > \
 451                             max_len.cores:
 452                         max_len.cores = \
 453                             len(failed_tests[test][e_message][u"cores"])
 454
 455             for test, message in failed_tests.items():
 456                 test_added = False
 457                 for e_message, params in message.items():
 458                     if not test_added:
 459                         test_added = True
 460                     else:
 461                         test = ""
 462                     text += (
 463                         f"{test + u' ' * (max_len.name - len(test))}  "
 464                         f"{params[u'nics']}"
 465                         f"{u' ' * (max_len.nics - len(params[u'nics']))}  "
 466                         f"{params[u'framesizes']}"
 467                         f"""{u' ' * (max_len.frmsizes
 468                                      - len(params[u'framesizes']))}  """
 469                         f"{params[u'cores']}"
 470                         f"{u' ' * (max_len.cores - len(params[u'cores']))}\n"
 471                     )
 472
 473             gression_hdr = (
 474                 f"\n\n{test_set_short}, "
 475                 f"CSIT build: {alert[u'urls'][idx]}/{build}, "
 476                 f"{device} version: {version}\n\n"
 477             )
 478             # Add list of regressions:
 479             self._list_gressions(alert, idx, gression_hdr, u"regressions")
 480
 481             # Add list of progressions:
 482             self._list_gressions(alert, idx, gression_hdr, u"progressions")
 483
 484         text += f"\nFor detailed information visit: {alert[u'url-details']}\n"
 485         file_name = f"{self.configs[alert[u'way']][u'output-dir']}/" \
 486                     f"{self.configs[alert[u'way']][u'output-file']}"
 487         logging.info(f"Writing the file {file_name}.txt ...")
 488
 489         text += f"\n\nLegend:\n\n"
 490
 491         for e_msg in self.error_msgs:
 492             text += f"[{self.error_msgs.index(e_msg)}] - {e_msg}\n"
 493
 494         try:
 495             with open(f"{file_name}.txt", u'w') as txt_file:
 496                 txt_file.write(text)
 497         except IOError:
 498             logging.error(f"Not possible to write the file {file_name}.txt.")