resources/tools/presentation/generator_alerts.py

   1 # Copyright (c) 2019 Cisco and/or its affiliates.
   2 # Licensed under the Apache License, Version 2.0 (the "License");
   3 # you may not use this file except in compliance with the License.
   4 # You may obtain a copy of the License at:
   5 #
   6 #     http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 # Unless required by applicable law or agreed to in writing, software
   9 # distributed under the License is distributed on an "AS IS" BASIS,
  10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 # See the License for the specific language governing permissions and
  12 # limitations under the License.
  13
  14 """Generator of alerts:
  15 - failed tests
  16 - regressions
  17 - progressions
  18 """
  19
  20
  21 import smtplib
  22 import logging
  23
  24 from email.mime.text import MIMEText
  25 from email.mime.multipart import MIMEMultipart
  26 from os.path import isdir
  27 from collections import OrderedDict
  28
  29 from pal_utils import get_last_completed_build_number
  30 from pal_errors import PresentationError
  31
  32
  33 class AlertingError(PresentationError):
  34     """Exception(s) raised by the alerting module.
  35
  36     When raising this exception, put this information to the message in this
  37     order:
  38      - short description of the encountered problem (parameter msg),
  39      - relevant messages if there are any collected, e.g., from caught
  40        exception (optional parameter details),
  41      - relevant data if there are any collected (optional parameter details).
  42     """
  43
  44     def __init__(self, msg, details=u'', level=u"CRITICAL"):
  45         """Sets the exception message and the level.
  46
  47         :param msg: Short description of the encountered problem.
  48         :param details: Relevant messages if there are any collected, e.g.,
  49             from caught exception (optional parameter details), or relevant data
  50             if there are any collected (optional parameter details).
  51         :param level: Level of the error, possible choices are: "DEBUG", "INFO",
  52             "WARNING", "ERROR" and "CRITICAL".
  53         :type msg: str
  54         :type details: str
  55         :type level: str
  56         """
  57
  58         super(AlertingError, self).__init__(f"Alerting: {msg}", details, level)
  59
  60     def __repr__(self):
  61         return (
  62             f"AlertingError(msg={self._msg!r},details={self._details!r},"
  63             f"level={self._level!r})"
  64         )
  65
  66
  67 class Alerting:
  68     """Class implementing the alerting mechanism.
  69     """
  70
  71     def __init__(self, spec):
  72         """Initialization.
  73
  74         :param spec: The CPTA specification.
  75         :type spec: Specification
  76         """
  77
  78         # Implemented alerts:
  79         self._implemented_alerts = (u"failed-tests", )
  80
  81         self._spec = spec
  82
  83         try:
  84             self._spec_alert = spec.alerting
  85         except KeyError as err:
  86             raise AlertingError(u"Alerting is not configured, skipped.",
  87                                 repr(err),
  88                                 u"WARNING")
  89
  90         self._path_failed_tests = spec.environment[u"paths"][u"DIR[STATIC,VPP]"]
  91
  92         # Verify and validate input specification:
  93         self.configs = self._spec_alert.get(u"configurations", None)
  94         if not self.configs:
  95             raise AlertingError(u"No alert configuration is specified.")
  96         for config_type, config_data in self.configs.items():
  97             if config_type == u"email":
  98                 if not config_data.get(u"server", None):
  99                     raise AlertingError(u"Parameter 'server' is missing.")
 100                 if not config_data.get(u"address-to", None):
 101                     raise AlertingError(u"Parameter 'address-to' (recipient) "
 102                                         u"is missing.")
 103                 if not config_data.get(u"address-from", None):
 104                     raise AlertingError(u"Parameter 'address-from' (sender) is "
 105                                         u"missing.")
 106             elif config_type == u"jenkins":
 107                 if not isdir(config_data.get(u"output-dir", u"")):
 108                     raise AlertingError(u"Parameter 'output-dir' is "
 109                                         u"missing or it is not a directory.")
 110                 if not config_data.get(u"output-file", None):
 111                     raise AlertingError(u"Parameter 'output-file' is missing.")
 112             else:
 113                 raise AlertingError(
 114                     f"Alert of type {config_type} is not implemented."
 115                 )
 116
 117         self.alerts = self._spec_alert.get(u"alerts", None)
 118         if not self.alerts:
 119             raise AlertingError(u"No alert is specified.")
 120         for alert_data in self.alerts.values():
 121             if not alert_data.get(u"title", None):
 122                 raise AlertingError(u"Parameter 'title' is missing.")
 123             if not alert_data.get(u"type", None) in self._implemented_alerts:
 124                 raise AlertingError(u"Parameter 'failed-tests' is missing or "
 125                                     u"incorrect.")
 126             if not alert_data.get(u"way", None) in self.configs.keys():
 127                 raise AlertingError(u"Parameter 'way' is missing or incorrect.")
 128             if not alert_data.get(u"include", None):
 129                 raise AlertingError(u"Parameter 'include' is missing or the "
 130                                     u"list is empty.")
 131
 132     def __str__(self):
 133         """Return string with human readable description of the alert.
 134
 135         :returns: Readable description.
 136         :rtype: str
 137         """
 138         return f"configs={self.configs}, alerts={self.alerts}"
 139
 140     def __repr__(self):
 141         """Return string executable as Python constructor call.
 142
 143         :returns: Executable constructor call.
 144         :rtype: str
 145         """
 146         return f"Alerting(spec={self._spec})"
 147
 148     def generate_alerts(self):
 149         """Generate alert(s) using specified way(s).
 150         """
 151
 152         for alert_data in self.alerts.values():
 153             if alert_data[u"way"] == u"jenkins":
 154                 self._generate_email_body(alert_data)
 155             else:
 156                 raise AlertingError(
 157                     f"Alert with way {alert_data[u'way']} is not implemented."
 158                 )
 159
 160     @staticmethod
 161     def _send_email(server, addr_from, addr_to, subject, text=None, html=None):
 162         """Send an email using predefined configuration.
 163
 164         :param server: SMTP server used to send email.
 165         :param addr_from: Sender address.
 166         :param addr_to: Recipient address(es).
 167         :param subject: Subject of the email.
 168         :param text: Message in the ASCII text format.
 169         :param html: Message in the HTML format.
 170         :type server: str
 171         :type addr_from: str
 172         :type addr_to: list
 173         :type subject: str
 174         :type text: str
 175         :type html: str
 176         """
 177
 178         if not text and not html:
 179             raise AlertingError(u"No text/data to send.")
 180
 181         msg = MIMEMultipart(u'alternative')
 182         msg[u'Subject'] = subject
 183         msg[u'From'] = addr_from
 184         msg[u'To'] = u", ".join(addr_to)
 185
 186         if text:
 187             msg.attach(MIMEText(text, u'plain'))
 188         if html:
 189             msg.attach(MIMEText(html, u'html'))
 190
 191         smtp_server = None
 192         try:
 193             logging.info(f"Trying to send alert {subject} ...")
 194             logging.debug(f"SMTP Server: {server}")
 195             logging.debug(f"From: {addr_from}")
 196             logging.debug(f"To: {u', '.join(addr_to)}")
 197             logging.debug(f"Message: {msg.as_string()}")
 198             smtp_server = smtplib.SMTP(server)
 199             smtp_server.sendmail(addr_from, addr_to, msg.as_string())
 200         except smtplib.SMTPException as err:
 201             raise AlertingError(u"Not possible to send the alert via email.",
 202                                 str(err))
 203         finally:
 204             if smtp_server:
 205                 smtp_server.quit()
 206
 207     def _get_compressed_failed_tests(self, alert, test_set, sort=True):
 208         """Return the dictionary with compressed faild tests. The compression is
 209         done by grouping the tests from the same area but with different NICs,
 210         frame sizes and number of processor cores.
 211
 212         For example, the failed tests:
 213           10ge2p1x520-64b-1c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
 214           10ge2p1x520-64b-2c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
 215           10ge2p1x520-64b-4c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
 216           10ge2p1x520-imix-1c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
 217           10ge2p1x520-imix-2c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
 218           10ge2p1x520-imix-4c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
 219
 220         will be represented as:
 221           ethip4udp-ip4scale4000-udpsrcscale15-nat44 \
 222           (10ge2p1x520, 64b, imix, 1c, 2c, 4c)
 223
 224         Structure of returned data:
 225
 226         {
 227             "trimmed_TC_name_1": {
 228                 "nics": [],
 229                 "framesizes": [],
 230                 "cores": []
 231             }
 232             ...
 233             "trimmed_TC_name_N": {
 234                 "nics": [],
 235                 "framesizes": [],
 236                 "cores": []
 237             }
 238         }
 239
 240         :param alert: Files are created for this alert.
 241         :param test_set: Specifies which set of tests will be included in the
 242             result. Its name is the same as the name of file with failed tests.
 243         :param sort: If True, the failed tests are sorted alphabetically.
 244         :type alert: dict
 245         :type test_set: str
 246         :type sort: bool
 247         :returns: CSIT build number, VPP version, Number of passed tests,
 248             Number of failed tests, Compressed failed tests.
 249         :rtype: tuple(str, str, int, int, OrderedDict)
 250         """
 251
 252         directory = self.configs[alert[u"way"]][u"output-dir"]
 253         failed_tests = OrderedDict()
 254         file_path = f"{directory}/{test_set}.txt"
 255         version = u""
 256         try:
 257             with open(file_path, u'r') as f_txt:
 258                 for idx, line in enumerate(f_txt):
 259                     if idx == 0:
 260                         build = line[:-1]
 261                         continue
 262                     if idx == 1:
 263                         version = line[:-1]
 264                         continue
 265                     if idx == 2:
 266                         passed = line[:-1]
 267                         continue
 268                     if idx == 3:
 269                         failed = line[:-1]
 270                         continue
 271                     try:
 272                         test = line[:-1].split(u'-')
 273                         name = u'-'.join(test[3:-1])
 274                     except IndexError:
 275                         continue
 276                     if failed_tests.get(name, None) is None:
 277                         failed_tests[name] = dict(nics=list(),
 278                                                   framesizes=list(),
 279                                                   cores=list())
 280                     if test[0] not in failed_tests[name][u"nics"]:
 281                         failed_tests[name][u"nics"].append(test[0])
 282                     if test[1] not in failed_tests[name][u"framesizes"]:
 283                         failed_tests[name][u"framesizes"].append(test[1])
 284                     if test[2] not in failed_tests[name][u"cores"]:
 285                         failed_tests[name][u"cores"].append(test[2])
 286         except IOError:
 287             logging.error(f"No such file or directory: {file_path}")
 288             return None, None, None, None, None
 289         if sort:
 290             sorted_failed_tests = OrderedDict()
 291             for key in sorted(failed_tests.keys()):
 292                 sorted_failed_tests[key] = failed_tests[key]
 293             return build, version, passed, failed, sorted_failed_tests
 294
 295         return build, version, passed, failed, failed_tests
 296
 297     def _list_gressions(self, alert, idx, header, re_pro):
 298         """Create a file with regressions or progressions for the test set
 299         specified by idx.
 300
 301         :param alert: Files are created for this alert.
 302         :param idx: Index of the test set as it is specified in the
 303             specification file.
 304         :param header: The header of the list of [re|pro]gressions.
 305         :param re_pro: 'regression' or 'progression'.
 306         :type alert: dict
 307         :type idx: int
 308         :type header: str
 309         :type re_pro: str
 310         """
 311
 312         if re_pro not in (u"regressions", u"progressions"):
 313             return
 314
 315         in_file = (
 316             f"{self.configs[alert[u'way']][u'output-dir']}/"
 317             f"{re_pro}-{alert[u'urls'][idx].split(u'/')[-1]}.txt"
 318         )
 319         out_file = (
 320             f"{self.configs[alert[u'way']][u'output-dir']}/"
 321             f"trending-{re_pro}.txt"
 322         )
 323
 324         try:
 325             with open(in_file, u'r') as txt_file:
 326                 file_content = txt_file.read()
 327                 with open(out_file, u'a+') as reg_file:
 328                     reg_file.write(header)
 329                     if file_content:
 330                         reg_file.write(file_content)
 331                     else:
 332                         reg_file.write(f"No {re_pro}")
 333         except IOError as err:
 334             logging.warning(repr(err))
 335
 336     def _generate_email_body(self, alert):
 337         """Create the file which is used in the generated alert.
 338
 339         :param alert: Files are created for this alert.
 340         :type alert: dict
 341         """
 342
 343         if alert[u"type"] != u"failed-tests":
 344             raise AlertingError(
 345                 f"Alert of type {alert[u'type']} is not implemented."
 346             )
 347
 348         text = u""
 349         for idx, test_set in enumerate(alert.get(u"include", [])):
 350             build, version, passed, failed, failed_tests = \
 351                 self._get_compressed_failed_tests(alert, test_set)
 352             if build is None:
 353                 ret_code, build_nr, _ = get_last_completed_build_number(
 354                     self._spec.environment[u"urls"][u"URL[JENKINS,CSIT]"],
 355                     alert[u"urls"][idx].split(u'/')[-1])
 356                 if ret_code != 0:
 357                     build_nr = u''
 358                 text += (
 359                     f"\n\nNo input data available for "
 360                     f"{u'-'.join(test_set.split('-')[-2:])}. See CSIT build "
 361                     f"{alert[u'urls'][idx]}/{build_nr} for more information.\n"
 362                 )
 363                 continue
 364             text += (
 365                 f"\n\n{test_set.split('-')[-2]}-{test_set.split('-')[-1]}, "
 366                 f"{failed} tests failed, "
 367                 f"{passed} tests passed, CSIT build: "
 368                 f"{alert[u'urls'][idx]}/{build}, VPP version: {version}\n\n"
 369             )
 370
 371             class MaxLens():
 372                 """Class to store the max lengths of strings displayed in
 373                 failed tests list.
 374                 """
 375                 def __init__(self, tst_name, nics, framesizes, cores):
 376                     """Initialisation.
 377
 378                     :param tst_name: Name of the test.
 379                     :param nics: NICs used in the test.
 380                     :param framesizes: Frame sizes used in the tests
 381                     :param cores: Cores used in th test.
 382                     """
 383                     self.name = tst_name
 384                     self.nics = nics
 385                     self.frmsizes = framesizes
 386                     self.cores = cores
 387
 388             max_len = MaxLens(0, 0, 0, 0)
 389
 390             for name, params in failed_tests.items():
 391                 failed_tests[name][u"nics"] = u",".join(sorted(params[u"nics"]))
 392                 failed_tests[name][u"framesizes"] = \
 393                     u",".join(sorted(params[u"framesizes"]))
 394                 failed_tests[name][u"cores"] = \
 395                     u",".join(sorted(params[u"cores"]))
 396                 if len(name) > max_len.name:
 397                     max_len.name = len(name)
 398                 if len(failed_tests[name][u"nics"]) > max_len.nics:
 399                     max_len.nics = len(failed_tests[name][u"nics"])
 400                 if len(failed_tests[name][u"framesizes"]) > max_len.frmsizes:
 401                     max_len.frmsizes = len(failed_tests[name][u"framesizes"])
 402                 if len(failed_tests[name][u"cores"]) > max_len.cores:
 403                     max_len.cores = len(failed_tests[name][u"cores"])
 404
 405             for name, params in failed_tests.items():
 406                 text += (
 407                     f"{name + u' ' * (max_len.name - len(name))}  "
 408                     f"{params[u'nics']}"
 409                     f"{u' ' * (max_len.nics - len(params[u'nics']))}  "
 410                     f"{params[u'framesizes']}"
 411                     f"{u' ' * (max_len.frmsizes-len(params[u'framesizes']))}  "
 412                     f"{params[u'cores']}"
 413                     f"{u' ' * (max_len.cores - len(params[u'cores']))}\n"
 414                 )
 415
 416             gression_hdr = (
 417                 f"\n\n{test_set.split(u'-')[-2]}-{test_set.split(u'-')[-1]}, "
 418                 f"CSIT build: {alert[u'urls'][idx]}/{build}, "
 419                 f"VPP version: {version}\n\n"
 420             )
 421             # Add list of regressions:
 422             self._list_gressions(alert, idx, gression_hdr, u"regressions")
 423
 424             # Add list of progressions:
 425             self._list_gressions(alert, idx, gression_hdr, u"progressions")
 426
 427         text += f"\nFor detailed information visit: {alert[u'url-details']}\n"
 428         file_name = f"{self.configs[alert[u'way']][u'output-dir']}/" \
 429                     f"{self.configs[alert[u'way']][u'output-file']}"
 430         logging.info(f"Writing the file {file_name}.txt ...")
 431
 432         try:
 433             with open(f"{file_name}.txt", u'w') as txt_file:
 434                 txt_file.write(text)
 435         except IOError:
 436             logging.error(f"Not possible to write the file {file_name}.txt.")