resources/tools/presentation/generator_alerts.py

   1 # Copyright (c) 2021 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Generator of alerts:
  15 - failed tests
  16 - regressions
  17 - progressions
  18 """
  19
  20
  21 import smtplib
  22 import logging
  23 import re
  24
  25 from difflib import SequenceMatcher
  26 from email.mime.text import MIMEText
  27 from email.mime.multipart import MIMEMultipart
  28 from os.path import isdir
  29 from collections import OrderedDict, defaultdict
  30
  31 from pal_errors import PresentationError
  32
  33
  34 class AlertingError(PresentationError):
  35     """Exception(s) raised by the alerting module.
  36
  37     When raising this exception, put this information to the message in this
  38     order:
  39      - short description of the encountered problem (parameter msg),
  40      - relevant messages if there are any collected, e.g., from caught
  41        exception (optional parameter details),
  42      - relevant data if there are any collected (optional parameter details).
  43     """
  44
  45     def __init__(self, msg, details=u'', level=u"CRITICAL"):
  46         """Sets the exception message and the level.
  47
  48         :param msg: Short description of the encountered problem.
  49         :param details: Relevant messages if there are any collected, e.g.,
  50             from caught exception (optional parameter details), or relevant data
  51             if there are any collected (optional parameter details).
  52         :param level: Level of the error, possible choices are: "DEBUG", "INFO",
  53             "WARNING", "ERROR" and "CRITICAL".
  54         :type msg: str
  55         :type details: str
  56         :type level: str
  57         """
  58
  59         super(AlertingError, self).__init__(f"Alerting: {msg}", details, level)
  60
  61     def __repr__(self):
  62         return (
  63             f"AlertingError(msg={self._msg!r},details={self._details!r},"
  64             f"level={self._level!r})"
  65         )
  66
  67
  68 class Alerting:
  69     """Class implementing the alerting mechanism.
  70     """
  71
  72     def __init__(self, spec):
  73         """Initialization.
  74
  75         :param spec: The CPTA specification.
  76         :type spec: Specification
  77         """
  78
  79         # Implemented alerts:
  80         self._implemented_alerts = (u"failed-tests", )
  81
  82         self._spec = spec
  83
  84         self.error_msgs = list()
  85
  86         try:
  87             self._spec_alert = spec.alerting
  88         except KeyError as err:
  89             raise AlertingError(
  90                 u"Alerting is not configured, skipped.", repr(err), u"WARNING"
  91             )
  92
  93         self._path_failed_tests = spec.environment[u"paths"][u"DIR[STATIC,VPP]"]
  94
  95         # Verify and validate input specification:
  96         self.configs = self._spec_alert.get(u"configurations", None)
  97         if not self.configs:
  98             raise AlertingError(u"No alert configuration is specified.")
  99         for config_type, config_data in self.configs.items():
 100             if config_type == u"email":
 101                 if not config_data.get(u"server", None):
 102                     raise AlertingError(u"Parameter 'server' is missing.")
 103                 if not config_data.get(u"address-to", None):
 104                     raise AlertingError(u"Parameter 'address-to' (recipient) "
 105                                         u"is missing.")
 106                 if not config_data.get(u"address-from", None):
 107                     raise AlertingError(u"Parameter 'address-from' (sender) is "
 108                                         u"missing.")
 109             elif config_type == u"jenkins":
 110                 if not isdir(config_data.get(u"output-dir", u"")):
 111                     raise AlertingError(u"Parameter 'output-dir' is "
 112                                         u"missing or it is not a directory.")
 113                 if not config_data.get(u"output-file", None):
 114                     raise AlertingError(u"Parameter 'output-file' is missing.")
 115             else:
 116                 raise AlertingError(
 117                     f"Alert of type {config_type} is not implemented."
 118                 )
 119
 120         self.alerts = self._spec_alert.get(u"alerts", None)
 121         if not self.alerts:
 122             raise AlertingError(u"No alert is specified.")
 123         for alert_data in self.alerts.values():
 124             if not alert_data.get(u"title", None):
 125                 raise AlertingError(u"Parameter 'title' is missing.")
 126             if not alert_data.get(u"type", None) in self._implemented_alerts:
 127                 raise AlertingError(u"Parameter 'failed-tests' is missing or "
 128                                     u"incorrect.")
 129             if not alert_data.get(u"way", None) in self.configs.keys():
 130                 raise AlertingError(u"Parameter 'way' is missing or incorrect.")
 131             if not alert_data.get(u"include", None):
 132                 raise AlertingError(u"Parameter 'include' is missing or the "
 133                                     u"list is empty.")
 134
 135     def __str__(self):
 136         """Return string with human readable description of the alert.
 137
 138         :returns: Readable description.
 139         :rtype: str
 140         """
 141         return f"configs={self.configs}, alerts={self.alerts}"
 142
 143     def __repr__(self):
 144         """Return string executable as Python constructor call.
 145
 146         :returns: Executable constructor call.
 147         :rtype: str
 148         """
 149         return f"Alerting(spec={self._spec})"
 150
 151     def generate_alerts(self):
 152         """Generate alert(s) using specified way(s).
 153         """
 154
 155         for alert_data in self.alerts.values():
 156             if alert_data[u"way"] == u"jenkins":
 157                 self._generate_email_body(alert_data)
 158             else:
 159                 raise AlertingError(
 160                     f"Alert with way {alert_data[u'way']} is not implemented."
 161                 )
 162
 163     @staticmethod
 164     def _send_email(server, addr_from, addr_to, subject, text=None, html=None):
 165         """Send an email using predefined configuration.
 166
 167         :param server: SMTP server used to send email.
 168         :param addr_from: Sender address.
 169         :param addr_to: Recipient address(es).
 170         :param subject: Subject of the email.
 171         :param text: Message in the ASCII text format.
 172         :param html: Message in the HTML format.
 173         :type server: str
 174         :type addr_from: str
 175         :type addr_to: list
 176         :type subject: str
 177         :type text: str
 178         :type html: str
 179         """
 180
 181         if not text and not html:
 182             raise AlertingError(u"No text/data to send.")
 183
 184         msg = MIMEMultipart(u'alternative')
 185         msg[u'Subject'] = subject
 186         msg[u'From'] = addr_from
 187         msg[u'To'] = u", ".join(addr_to)
 188
 189         if text:
 190             msg.attach(MIMEText(text, u'plain'))
 191         if html:
 192             msg.attach(MIMEText(html, u'html'))
 193
 194         smtp_server = None
 195         try:
 196             logging.info(f"Trying to send alert {subject} ...")
 197             logging.debug(f"SMTP Server: {server}")
 198             logging.debug(f"From: {addr_from}")
 199             logging.debug(f"To: {u', '.join(addr_to)}")
 200             logging.debug(f"Message: {msg.as_string()}")
 201             smtp_server = smtplib.SMTP(server)
 202             smtp_server.sendmail(addr_from, addr_to, msg.as_string())
 203         except smtplib.SMTPException as err:
 204             raise AlertingError(u"Not possible to send the alert via email.",
 205                                 str(err))
 206         finally:
 207             if smtp_server:
 208                 smtp_server.quit()
 209
 210     def _get_compressed_failed_tests(self, alert, test_set, sort=True):
 211         """Return the dictionary with compressed faild tests. The compression is
 212         done by grouping the tests from the same area but with different NICs,
 213         frame sizes and number of processor cores.
 214
 215         For example, the failed tests:
 216           10ge2p1x520-64b-1c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
 217           10ge2p1x520-64b-2c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
 218           10ge2p1x520-64b-4c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
 219           10ge2p1x520-imix-1c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
 220           10ge2p1x520-imix-2c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
 221           10ge2p1x520-imix-4c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
 222
 223         will be represented as:
 224           ethip4udp-ip4scale4000-udpsrcscale15-nat44 \
 225           (10ge2p1x520, 64b, imix, 1c, 2c, 4c)
 226
 227         Structure of returned data:
 228
 229         {
 230             "trimmed_TC_name_1": {
 231                 "nics": [],
 232                 "framesizes": [],
 233                 "cores": []
 234             }
 235             ...
 236             "trimmed_TC_name_N": {
 237                 "nics": [],
 238                 "framesizes": [],
 239                 "cores": []
 240             }
 241         }
 242
 243         :param alert: Files are created for this alert.
 244         :param test_set: Specifies which set of tests will be included in the
 245             result. Its name is the same as the name of file with failed tests.
 246         :param sort: If True, the failed tests are sorted alphabetically.
 247         :type alert: dict
 248         :type test_set: str
 249         :type sort: bool
 250         :returns: CSIT build number, VPP version, Number of passed tests,
 251             Number of failed tests, Compressed failed tests.
 252         :rtype: tuple(str, str, int, int, str, OrderedDict)
 253         """
 254
 255         directory = self.configs[alert[u"way"]][u"output-dir"]
 256         failed_tests = defaultdict(dict)
 257         file_path = f"{directory}/{test_set}.txt"
 258         version = u""
 259         try:
 260             with open(file_path, u'r') as f_txt:
 261                 for idx, line in enumerate(f_txt):
 262                     if idx == 0:
 263                         build = line[:-1]
 264                         continue
 265                     if idx == 1:
 266                         version = line[:-1]
 267                         continue
 268                     if idx == 2:
 269                         passed = line[:-1]
 270                         continue
 271                     if idx == 3:
 272                         failed = line[:-1]
 273                         continue
 274                     if idx == 4:
 275                         minutes = int(line[:-1]) // 60000
 276                         duration = f"{(minutes // 60):02d}:{(minutes % 60):02d}"
 277                         continue
 278                     try:
 279                         line, error_msg = line[:-1].split(u'###', maxsplit=1)
 280                         test = line.split(u'-')
 281                         name = u'-'.join(test[3:-1])
 282                     except ValueError:
 283                         continue
 284
 285                     for e_msg in self.error_msgs:
 286                         if SequenceMatcher(None, e_msg,
 287                                            error_msg).ratio() > 0.5:
 288                             error_msg = e_msg
 289                             break
 290                     if error_msg not in self.error_msgs:
 291                         self.error_msgs.append(error_msg)
 292
 293                     error_msg_index = self.error_msgs.index(error_msg)
 294
 295                     if failed_tests.get(name, {}).get(error_msg_index) is None:
 296                         failed_tests[name][error_msg_index] = \
 297                             dict(nics=list(),
 298                                  framesizes=list(),
 299                                  cores=list())
 300
 301                     if test[0] not in \
 302                             failed_tests[name][error_msg_index][u"nics"]:
 303                         failed_tests[name][error_msg_index][u"nics"].\
 304                             append(test[0])
 305                     if test[1] not in \
 306                             failed_tests[name][error_msg_index][u"framesizes"]:
 307                         failed_tests[name][error_msg_index][u"framesizes"].\
 308                             append(test[1])
 309                     check_core = test[2] + f"[{str(error_msg_index)}]"
 310                     if check_core not in \
 311                             failed_tests[name][error_msg_index][u"cores"]:
 312                         failed_tests[name][error_msg_index][u"cores"].\
 313                             append(test[2] + "[" + str(error_msg_index) + "]")
 314
 315         except IOError:
 316             logging.error(f"No such file or directory: {file_path}")
 317             return None, None, None, None, None, None
 318         if sort:
 319             sorted_failed_tests = OrderedDict()
 320             for key in sorted(failed_tests.keys()):
 321                 sorted_failed_tests[key] = failed_tests[key]
 322             return build, version, passed, failed, duration, sorted_failed_tests
 323
 324         return build, version, passed, failed, duration, failed_tests
 325
 326     def _list_gressions(self, alert, idx, header, re_pro):
 327         """Create a file with regressions or progressions for the test set
 328         specified by idx.
 329
 330         :param alert: Files are created for this alert.
 331         :param idx: Index of the test set as it is specified in the
 332             specification file.
 333         :param header: The header of the list of [re|pro]gressions.
 334         :param re_pro: 'regression' or 'progression'.
 335         :type alert: dict
 336         :type idx: int
 337         :type header: str
 338         :type re_pro: str
 339         """
 340
 341         if re_pro not in (u"regressions", u"progressions"):
 342             return
 343
 344         in_file = (
 345             f"{self.configs[alert[u'way']][u'output-dir']}/"
 346             f"{re_pro}-{alert[u'urls'][idx].split(u'/')[-1]}.txt"
 347         )
 348         out_file = (
 349             f"{self.configs[alert[u'way']][u'output-dir']}/"
 350             f"trending-{re_pro}.txt"
 351         )
 352
 353         try:
 354             with open(in_file, u'r') as txt_file:
 355                 file_content = txt_file.read()
 356                 with open(out_file, u'a+') as reg_file:
 357                     reg_file.write(header)
 358                     if file_content:
 359                         reg_file.write(file_content)
 360                     else:
 361                         reg_file.write(f"No {re_pro}")
 362         except IOError as err:
 363             logging.warning(repr(err))
 364
 365     def _generate_email_body(self, alert):
 366         """Create the file which is used in the generated alert.
 367
 368         :param alert: Files are created for this alert.
 369         :type alert: dict
 370         """
 371
 372         if alert[u"type"] != u"failed-tests":
 373             raise AlertingError(
 374                 f"Alert of type {alert[u'type']} is not implemented."
 375             )
 376
 377         text = u""
 378
 379         legend = (f"Legend:\n[ Last trend in Mpps | number of runs for "
 380                   f"last trend |")
 381
 382         out_file = (
 383             f"{self.configs[alert[u'way']][u'output-dir']}/"
 384             f"trending-regressions.txt"
 385         )
 386         try:
 387             with open(out_file, u'w') as reg_file:
 388                 reg_file.write(f"{legend} regressions ]")
 389         except IOError:
 390             logging.error(f"Not possible to write the file {out_file}.txt.")
 391
 392         out_file = (
 393             f"{self.configs[alert[u'way']][u'output-dir']}/"
 394             f"trending-progressions.txt"
 395         )
 396         try:
 397             with open(out_file, u'w') as reg_file:
 398                 reg_file.write(f"{legend} progressions ]")
 399         except IOError:
 400             logging.error(f"Not possible to write the file {out_file}.txt.")
 401
 402         for idx, test_set in enumerate(alert.get(u"include", list())):
 403             test_set_short = u""
 404             device = u""
 405             try:
 406                 groups = re.search(
 407                     re.compile(
 408                         r'((vpp|dpdk)-\dn-(skx|clx|tsh|dnv|zn2|tx2)-.*)'
 409                     ),
 410                     test_set
 411                 )
 412                 test_set_short = groups.group(1)
 413                 device = groups.group(2)
 414             except (AttributeError, IndexError):
 415                 logging.error(
 416                     f"The test set {test_set} does not include information "
 417                     f"about test bed. Using empty string instead."
 418                 )
 419             build, version, passed, failed, duration, failed_tests = \
 420                 self._get_compressed_failed_tests(alert, test_set)
 421             if build is None:
 422                 text += (
 423                     f"\n\nNo input data available for {test_set_short}. "
 424                     f"See CSIT job {alert[u'urls'][idx]} for more "
 425                     f"information.\n"
 426                 )
 427                 continue
 428             text += (
 429                 f"\n\n{test_set_short}, "
 430                 f"{failed} tests failed, "
 431                 f"{passed} tests passed, "
 432                 f"duration: {duration}, "
 433                 f"CSIT build: {alert[u'urls'][idx]}/{build}, "
 434                 f"{device} version: {version}\n\n"
 435             )
 436
 437             class MaxLens():
 438                 """Class to store the max lengths of strings displayed in
 439                 failed tests list.
 440                 """
 441                 def __init__(self, tst_name, nics, framesizes, cores):
 442                     """Initialisation.
 443
 444                     :param tst_name: Name of the test.
 445                     :param nics: NICs used in the test.
 446                     :param framesizes: Frame sizes used in the tests
 447                     :param cores: Cores used in th test.
 448                     """
 449                     self.name = tst_name
 450                     self.nics = nics
 451                     self.frmsizes = framesizes
 452                     self.cores = cores
 453
 454             max_len = MaxLens(0, 0, 0, 0)
 455
 456             for test, message in failed_tests.items():
 457                 for e_message, params in message.items():
 458                     failed_tests[test][e_message][u"nics"] = \
 459                         u" ".join(sorted(params[u"nics"]))
 460                     failed_tests[test][e_message][u"framesizes"] = \
 461                         u" ".join(sorted(params[u"framesizes"]))
 462                     failed_tests[test][e_message][u"cores"] = \
 463                         u" ".join(sorted(params[u"cores"]))
 464                     if len(test) > max_len.name:
 465                         max_len.name = len(test)
 466                     if len(failed_tests[test][e_message][u"nics"]) > \
 467                             max_len.nics:
 468                         max_len.nics = \
 469                             len(failed_tests[test][e_message][u"nics"])
 470                     if len(failed_tests[test][e_message][u"framesizes"]) > \
 471                             max_len.frmsizes:
 472                         max_len.frmsizes = \
 473                             len(failed_tests[test][e_message][u"framesizes"])
 474                     if len(failed_tests[test][e_message][u"cores"]) > \
 475                             max_len.cores:
 476                         max_len.cores = \
 477                             len(failed_tests[test][e_message][u"cores"])
 478
 479             for test, message in failed_tests.items():
 480                 test_added = False
 481                 for e_message, params in message.items():
 482                     if not test_added:
 483                         test_added = True
 484                     else:
 485                         test = ""
 486                     text += (
 487                         f"{test + u' ' * (max_len.name - len(test))}  "
 488                         f"{params[u'nics']}"
 489                         f"{u' ' * (max_len.nics - len(params[u'nics']))}  "
 490                         f"{params[u'framesizes']}"
 491                         f"""{u' ' * (max_len.frmsizes
 492                                      - len(params[u'framesizes']))}  """
 493                         f"{params[u'cores']}"
 494                         f"{u' ' * (max_len.cores - len(params[u'cores']))}\n"
 495                     )
 496
 497             gression_hdr = (
 498                 f"\n\n{test_set_short}, "
 499                 f"CSIT build: {alert[u'urls'][idx]}/{build}, "
 500                 f"{device} version: {version}\n\n"
 501             )
 502             # Add list of regressions:
 503             self._list_gressions(alert, idx, gression_hdr, u"regressions")
 504
 505             # Add list of progressions:
 506             self._list_gressions(alert, idx, gression_hdr, u"progressions")
 507
 508         text += f"\nFor detailed information visit: {alert[u'url-details']}\n"
 509         file_name = f"{self.configs[alert[u'way']][u'output-dir']}/" \
 510                     f"{self.configs[alert[u'way']][u'output-file']}"
 511         logging.info(f"Writing the file {file_name}.txt ...")
 512
 513         text += f"\n\nLegend:\n\n"
 514
 515         for e_msg in self.error_msgs:
 516             text += f"[{self.error_msgs.index(e_msg)}] - {e_msg}\n"
 517
 518         try:
 519             with open(f"{file_name}.txt", u'w') as txt_file:
 520                 txt_file.write(text)
 521         except IOError:
 522             logging.error(f"Not possible to write the file {file_name}.txt.")