resources/tools/presentation/generator_alerts.py

   1 # Copyright (c) 2022 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Generator of alerts:
  15 - failed tests
  16 - regressions
  17 - progressions
  18 """
  19
  20
  21 import smtplib
  22 import logging
  23 import re
  24
  25 from difflib import SequenceMatcher
  26 from email.mime.text import MIMEText
  27 from email.mime.multipart import MIMEMultipart
  28 from os.path import isdir
  29 from collections import OrderedDict, defaultdict
  30
  31 from pal_errors import PresentationError
  32
  33
  34 class AlertingError(PresentationError):
  35     """Exception(s) raised by the alerting module.
  36
  37     When raising this exception, put this information to the message in this
  38     order:
  39      - short description of the encountered problem (parameter msg),
  40      - relevant messages if there are any collected, e.g., from caught
  41        exception (optional parameter details),
  42      - relevant data if there are any collected (optional parameter details).
  43     """
  44
  45     def __init__(self, msg, details=u'', level=u"CRITICAL"):
  46         """Sets the exception message and the level.
  47
  48         :param msg: Short description of the encountered problem.
  49         :param details: Relevant messages if there are any collected, e.g.,
  50             from caught exception (optional parameter details), or relevant data
  51             if there are any collected (optional parameter details).
  52         :param level: Level of the error, possible choices are: "DEBUG", "INFO",
  53             "WARNING", "ERROR" and "CRITICAL".
  54         :type msg: str
  55         :type details: str
  56         :type level: str
  57         """
  58
  59         super(AlertingError, self).__init__(f"Alerting: {msg}", details, level)
  60
  61     def __repr__(self):
  62         return (
  63             f"AlertingError(msg={self._msg!r},details={self._details!r},"
  64             f"level={self._level!r})"
  65         )
  66
  67
  68 class Alerting:
  69     """Class implementing the alerting mechanism.
  70     """
  71
  72     def __init__(self, spec):
  73         """Initialization.
  74
  75         :param spec: The CPTA specification.
  76         :type spec: Specification
  77         """
  78
  79         # Implemented alerts:
  80         self._implemented_alerts = (u"failed-tests", )
  81
  82         self._spec = spec
  83
  84         self.error_msgs = list()
  85
  86         try:
  87             self._spec_alert = spec.alerting
  88         except KeyError as err:
  89             raise AlertingError(
  90                 u"Alerting is not configured, skipped.", repr(err), u"WARNING"
  91             )
  92
  93         self._path_failed_tests = spec.environment[u"paths"][u"DIR[STATIC,VPP]"]
  94
  95         # Verify and validate input specification:
  96         self.configs = self._spec_alert.get(u"configurations", None)
  97         if not self.configs:
  98             raise AlertingError(u"No alert configuration is specified.")
  99         for config_type, config_data in self.configs.items():
 100             if config_type == u"email":
 101                 if not config_data.get(u"server", None):
 102                     raise AlertingError(u"Parameter 'server' is missing.")
 103                 if not config_data.get(u"address-to", None):
 104                     raise AlertingError(u"Parameter 'address-to' (recipient) "
 105                                         u"is missing.")
 106                 if not config_data.get(u"address-from", None):
 107                     raise AlertingError(u"Parameter 'address-from' (sender) is "
 108                                         u"missing.")
 109             elif config_type == u"jenkins":
 110                 if not isdir(config_data.get(u"output-dir", u"")):
 111                     raise AlertingError(u"Parameter 'output-dir' is "
 112                                         u"missing or it is not a directory.")
 113                 if not config_data.get(u"output-file", None):
 114                     raise AlertingError(u"Parameter 'output-file' is missing.")
 115             else:
 116                 raise AlertingError(
 117                     f"Alert of type {config_type} is not implemented."
 118                 )
 119
 120         self.alerts = self._spec_alert.get(u"alerts", None)
 121         if not self.alerts:
 122             raise AlertingError(u"No alert is specified.")
 123         for alert_data in self.alerts.values():
 124             if not alert_data.get(u"title", None):
 125                 raise AlertingError(u"Parameter 'title' is missing.")
 126             if not alert_data.get(u"type", None) in self._implemented_alerts:
 127                 raise AlertingError(u"Parameter 'failed-tests' is missing or "
 128                                     u"incorrect.")
 129             if not alert_data.get(u"way", None) in self.configs.keys():
 130                 raise AlertingError(u"Parameter 'way' is missing or incorrect.")
 131             if not alert_data.get(u"include", None):
 132                 raise AlertingError(u"Parameter 'include' is missing or the "
 133                                     u"list is empty.")
 134
 135     def __str__(self):
 136         """Return string with human readable description of the alert.
 137
 138         :returns: Readable description.
 139         :rtype: str
 140         """
 141         return f"configs={self.configs}, alerts={self.alerts}"
 142
 143     def __repr__(self):
 144         """Return string executable as Python constructor call.
 145
 146         :returns: Executable constructor call.
 147         :rtype: str
 148         """
 149         return f"Alerting(spec={self._spec})"
 150
 151     def generate_alerts(self):
 152         """Generate alert(s) using specified way(s).
 153         """
 154
 155         for alert_data in self.alerts.values():
 156             if alert_data[u"way"] == u"jenkins":
 157                 self._generate_email_body(alert_data)
 158             else:
 159                 raise AlertingError(
 160                     f"Alert with way {alert_data[u'way']} is not implemented."
 161                 )
 162
 163     @staticmethod
 164     def _send_email(server, addr_from, addr_to, subject, text=None, html=None):
 165         """Send an email using predefined configuration.
 166
 167         :param server: SMTP server used to send email.
 168         :param addr_from: Sender address.
 169         :param addr_to: Recipient address(es).
 170         :param subject: Subject of the email.
 171         :param text: Message in the ASCII text format.
 172         :param html: Message in the HTML format.
 173         :type server: str
 174         :type addr_from: str
 175         :type addr_to: list
 176         :type subject: str
 177         :type text: str
 178         :type html: str
 179         """
 180
 181         if not text and not html:
 182             raise AlertingError(u"No text/data to send.")
 183
 184         msg = MIMEMultipart(u'alternative')
 185         msg[u'Subject'] = subject
 186         msg[u'From'] = addr_from
 187         msg[u'To'] = u", ".join(addr_to)
 188
 189         if text:
 190             msg.attach(MIMEText(text, u'plain'))
 191         if html:
 192             msg.attach(MIMEText(html, u'html'))
 193
 194         smtp_server = None
 195         try:
 196             logging.info(f"Trying to send alert {subject} ...")
 197             logging.debug(f"SMTP Server: {server}")
 198             logging.debug(f"From: {addr_from}")
 199             logging.debug(f"To: {u', '.join(addr_to)}")
 200             logging.debug(f"Message: {msg.as_string()}")
 201             smtp_server = smtplib.SMTP(server)
 202             smtp_server.sendmail(addr_from, addr_to, msg.as_string())
 203         except smtplib.SMTPException as err:
 204             raise AlertingError(u"Not possible to send the alert via email.",
 205                                 str(err))
 206         finally:
 207             if smtp_server:
 208                 smtp_server.quit()
 209
 210     def _get_compressed_failed_tests(self, alert, test_set, sort=True):
 211         """Return the dictionary with compressed faild tests. The compression is
 212         done by grouping the tests from the same area but with different NICs,
 213         frame sizes and number of processor cores.
 214
 215         For example, the failed tests:
 216           10ge2p1x520-64b-1c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
 217           10ge2p1x520-64b-2c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
 218           10ge2p1x520-64b-4c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
 219           10ge2p1x520-imix-1c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
 220           10ge2p1x520-imix-2c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
 221           10ge2p1x520-imix-4c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
 222
 223         will be represented as:
 224           ethip4udp-ip4scale4000-udpsrcscale15-nat44 \
 225           (10ge2p1x520, 64b, imix, 1c, 2c, 4c)
 226
 227         Structure of returned data:
 228
 229         {
 230             "trimmed_TC_name_1": {
 231                 "nics": [],
 232                 "framesizes": [],
 233                 "cores": []
 234             }
 235             ...
 236             "trimmed_TC_name_N": {
 237                 "nics": [],
 238                 "framesizes": [],
 239                 "cores": []
 240             }
 241         }
 242
 243         :param alert: Files are created for this alert.
 244         :param test_set: Specifies which set of tests will be included in the
 245             result. Its name is the same as the name of file with failed tests.
 246         :param sort: If True, the failed tests are sorted alphabetically.
 247         :type alert: dict
 248         :type test_set: str
 249         :type sort: bool
 250         :returns: CSIT build number, VPP version, Number of passed tests,
 251             Number of failed tests, Compressed failed tests.
 252         :rtype: tuple(str, str, int, int, str, OrderedDict)
 253         """
 254
 255         directory = self.configs[alert[u"way"]][u"output-dir"]
 256         failed_tests = defaultdict(dict)
 257         file_path = f"{directory}/{test_set}.txt"
 258         version = u""
 259         try:
 260             with open(file_path, u'r') as f_txt:
 261                 for idx, line in enumerate(f_txt):
 262                     if idx == 0:
 263                         build = line[:-1]
 264                         continue
 265                     if idx == 1:
 266                         version = line[:-1]
 267                         continue
 268                     if idx == 2:
 269                         passed = line[:-1]
 270                         continue
 271                     if idx == 3:
 272                         failed = line[:-1]
 273                         continue
 274                     if idx == 4:
 275                         minutes = int(line[:-1]) // 60000
 276                         duration = f"{(minutes // 60):02d}:{(minutes % 60):02d}"
 277                         continue
 278                     try:
 279                         line, error_msg = line[:-1].split(u'###', maxsplit=1)
 280                         test = line.split(u'-')
 281                         name = u'-'.join(test[3:-1])
 282                         if len(error_msg) > 128:
 283                             if u";" in error_msg[128:256]:
 284                                 error_msg = \
 285                                     f"{error_msg[:128]}" \
 286                                     f"{error_msg[128:].split(u';', 1)[0]}..."
 287                             elif u":" in error_msg[128:256]:
 288                                 error_msg = \
 289                                     f"{error_msg[:128]}" \
 290                                     f"{error_msg[128:].split(u':', 1)[0]}..."
 291                             elif u"." in error_msg[128:256]:
 292                                 error_msg = \
 293                                     f"{error_msg[:128]}" \
 294                                     f"{error_msg[128:].split(u'.', 1)[0]}..."
 295                             elif u"?" in error_msg[128:256]:
 296                                 error_msg = \
 297                                     f"{error_msg[:128]}" \
 298                                     f"{error_msg[128:].split(u'?', 1)[0]}..."
 299                             elif u"!" in error_msg[128:256]:
 300                                 error_msg = \
 301                                     f"{error_msg[:128]}" \
 302                                     f"{error_msg[128:].split(u'!', 1)[0]}..."
 303                             elif u"," in error_msg[128:256]:
 304                                 error_msg = \
 305                                     f"{error_msg[:128]}" \
 306                                     f"{error_msg[128:].split(u',', 1)[0]}..."
 307                             elif u" " in error_msg[128:256]:
 308                                 error_msg = \
 309                                     f"{error_msg[:128]}" \
 310                                     f"{error_msg[128:].split(u' ', 1)[0]}..."
 311                             else:
 312                                 error_msg = error_msg[:128]
 313
 314                     except ValueError:
 315                         continue
 316
 317                     for e_msg in self.error_msgs:
 318                         if SequenceMatcher(None, e_msg,
 319                                            error_msg).ratio() > 0.5:
 320                             error_msg = e_msg
 321                             break
 322                     if error_msg not in self.error_msgs:
 323                         self.error_msgs.append(error_msg)
 324
 325                     error_msg_index = self.error_msgs.index(error_msg)
 326
 327                     if failed_tests.get(name, {}).get(error_msg_index) is None:
 328                         failed_tests[name][error_msg_index] = \
 329                             dict(nics=list(),
 330                                  framesizes=list(),
 331                                  cores=list())
 332
 333                     if test[0] not in \
 334                             failed_tests[name][error_msg_index][u"nics"]:
 335                         failed_tests[name][error_msg_index][u"nics"].\
 336                             append(test[0])
 337                     if test[1] not in \
 338                             failed_tests[name][error_msg_index][u"framesizes"]:
 339                         failed_tests[name][error_msg_index][u"framesizes"].\
 340                             append(test[1])
 341                     check_core = test[2] + f"[{str(error_msg_index)}]"
 342                     if check_core not in \
 343                             failed_tests[name][error_msg_index][u"cores"]:
 344                         failed_tests[name][error_msg_index][u"cores"].\
 345                             append(test[2] + "[" + str(error_msg_index) + "]")
 346
 347         except IOError:
 348             logging.error(f"No such file or directory: {file_path}")
 349             return None, None, None, None, None, None
 350         if sort:
 351             sorted_failed_tests = OrderedDict()
 352             for key in sorted(failed_tests.keys()):
 353                 sorted_failed_tests[key] = failed_tests[key]
 354             return build, version, passed, failed, duration, sorted_failed_tests
 355
 356         return build, version, passed, failed, duration, failed_tests
 357
 358     def _list_gressions(self, alert, idx, header, re_pro):
 359         """Create a file with regressions or progressions for the test set
 360         specified by idx.
 361
 362         :param alert: Files are created for this alert.
 363         :param idx: Index of the test set as it is specified in the
 364             specification file.
 365         :param header: The header of the list of [re|pro]gressions.
 366         :param re_pro: 'regressions' or 'progressions'.
 367         :type alert: dict
 368         :type idx: int
 369         :type header: str
 370         :type re_pro: str
 371         """
 372
 373         if re_pro not in (u"regressions", u"progressions"):
 374             return
 375
 376         in_file = (
 377             f"{self.configs[alert[u'way']][u'output-dir']}/"
 378             f"{re_pro}-{alert[u'urls'][idx].split(u'/')[-1]}.txt"
 379         )
 380         out_file = (
 381             f"{self.configs[alert[u'way']][u'output-dir']}/"
 382             f"trending-{re_pro}.txt"
 383         )
 384
 385         try:
 386             with open(in_file, u'r') as txt_file:
 387                 file_content = txt_file.read()
 388                 with open(out_file, u'a+') as reg_file:
 389                     reg_file.write(header)
 390                     if file_content:
 391                         reg_file.write(file_content)
 392                     else:
 393                         reg_file.write(f"No {re_pro}")
 394         except IOError as err:
 395             logging.warning(repr(err))
 396
 397     def _generate_email_body(self, alert):
 398         """Create the file which is used in the generated alert.
 399
 400         :param alert: Files are created for this alert.
 401         :type alert: dict
 402         """
 403
 404         if alert[u"type"] != u"failed-tests":
 405             raise AlertingError(
 406                 f"Alert of type {alert[u'type']} is not implemented."
 407             )
 408
 409         text = u""
 410
 411         legend = (f"Legend: Test-name  NIC  Frame-size  Trend[Mpps]  Runs[#]  "
 412                   f"Long-Term change[%]")
 413
 414         out_file = (
 415             f"{self.configs[alert[u'way']][u'output-dir']}/"
 416             f"trending-regressions.txt"
 417         )
 418         try:
 419             with open(out_file, u'w') as reg_file:
 420                 reg_file.write(legend)
 421         except IOError:
 422             logging.error(f"Not possible to write the file {out_file}.txt.")
 423
 424         out_file = (
 425             f"{self.configs[alert[u'way']][u'output-dir']}/"
 426             f"trending-progressions.txt"
 427         )
 428         try:
 429             with open(out_file, u'w') as reg_file:
 430                 reg_file.write(legend)
 431         except IOError:
 432             logging.error(f"Not possible to write the file {out_file}.txt.")
 433
 434         for idx, test_set in enumerate(alert.get(u"include", list())):
 435             test_set_short = u""
 436             device = u""
 437             try:
 438                 groups = re.search(
 439                     re.compile(
 440                         r'((vpp|dpdk)-\dn-(skx|clx|tsh|dnv|zn2|tx2|icx)-.*)'
 441                     ),
 442                     test_set
 443                 )
 444                 test_set_short = groups.group(1)
 445                 device = groups.group(2)
 446             except (AttributeError, IndexError):
 447                 logging.error(
 448                     f"The test set {test_set} does not include information "
 449                     f"about test bed. Using empty string instead."
 450                 )
 451             build, version, passed, failed, duration, failed_tests = \
 452                 self._get_compressed_failed_tests(alert, test_set)
 453             if build is None:
 454                 text += (
 455                     f"\n\nNo input data available for {test_set_short}. "
 456                     f"See CSIT job {alert[u'urls'][idx]} for more "
 457                     f"information.\n"
 458                 )
 459                 continue
 460             text += (
 461                 f"\n\n{test_set_short}, "
 462                 f"{failed} tests failed, "
 463                 f"{passed} tests passed, "
 464                 f"duration: {duration}, "
 465                 f"CSIT build: {alert[u'urls'][idx]}/{build}, "
 466                 f"{device} version: {version}\n\n"
 467             )
 468
 469             class MaxLens():
 470                 """Class to store the max lengths of strings displayed in
 471                 failed tests list.
 472                 """
 473                 def __init__(self, tst_name, nics, framesizes, cores):
 474                     """Initialisation.
 475
 476                     :param tst_name: Name of the test.
 477                     :param nics: NICs used in the test.
 478                     :param framesizes: Frame sizes used in the tests
 479                     :param cores: Cores used in th test.
 480                     """
 481                     self.name = tst_name
 482                     self.nics = nics
 483                     self.frmsizes = framesizes
 484                     self.cores = cores
 485
 486             max_len = MaxLens(0, 0, 0, 0)
 487
 488             for test, message in failed_tests.items():
 489                 for e_message, params in message.items():
 490                     failed_tests[test][e_message][u"nics"] = \
 491                         u" ".join(sorted(params[u"nics"]))
 492                     failed_tests[test][e_message][u"framesizes"] = \
 493                         u" ".join(sorted(params[u"framesizes"]))
 494                     failed_tests[test][e_message][u"cores"] = \
 495                         u" ".join(sorted(params[u"cores"]))
 496                     if len(test) > max_len.name:
 497                         max_len.name = len(test)
 498                     if len(failed_tests[test][e_message][u"nics"]) > \
 499                             max_len.nics:
 500                         max_len.nics = \
 501                             len(failed_tests[test][e_message][u"nics"])
 502                     if len(failed_tests[test][e_message][u"framesizes"]) > \
 503                             max_len.frmsizes:
 504                         max_len.frmsizes = \
 505                             len(failed_tests[test][e_message][u"framesizes"])
 506                     if len(failed_tests[test][e_message][u"cores"]) > \
 507                             max_len.cores:
 508                         max_len.cores = \
 509                             len(failed_tests[test][e_message][u"cores"])
 510
 511             for test, message in failed_tests.items():
 512                 test_added = False
 513                 for e_message, params in message.items():
 514                     if not test_added:
 515                         test_added = True
 516                     else:
 517                         test = ""
 518                     text += (
 519                         f"{test + u' ' * (max_len.name - len(test))}  "
 520                         f"{params[u'nics']}"
 521                         f"{u' ' * (max_len.nics - len(params[u'nics']))}  "
 522                         f"{params[u'framesizes']}"
 523                         f"""{u' ' * (max_len.frmsizes
 524                                      - len(params[u'framesizes']))}  """
 525                         f"{params[u'cores']}"
 526                         f"{u' ' * (max_len.cores - len(params[u'cores']))}\n"
 527                     )
 528
 529             gression_hdr = (
 530                 f"\n\n{test_set_short}, "
 531                 f"CSIT build: {alert[u'urls'][idx]}/{build}, "
 532                 f"{device} version: {version}\n\n"
 533             )
 534             # Add list of regressions:
 535             self._list_gressions(alert, idx, gression_hdr, u"regressions")
 536
 537             # Add list of progressions:
 538             self._list_gressions(alert, idx, gression_hdr, u"progressions")
 539
 540         text += f"\nFor detailed information visit: {alert[u'url-details']}\n"
 541         file_name = f"{self.configs[alert[u'way']][u'output-dir']}/" \
 542                     f"{self.configs[alert[u'way']][u'output-file']}"
 543         logging.info(f"Writing the file {file_name}.txt ...")
 544
 545         text += f"\n\nLegend:\n\n"
 546
 547         for e_msg in self.error_msgs:
 548             text += f"[{self.error_msgs.index(e_msg)}] - {e_msg}\n"
 549
 550         try:
 551             with open(f"{file_name}.txt", u'w') as txt_file:
 552                 txt_file.write(text)
 553         except IOError:
 554             logging.error(f"Not possible to write the file {file_name}.txt.")