Report: Add data
[csit.git] / resources / tools / presentation / generator_alerts.py
1 # Copyright (c) 2022 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
5 #
6 #     http://www.apache.org/licenses/LICENSE-2.0
7 #
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
13
14 """Generator of alerts:
15 - failed tests
16 - regressions
17 - progressions
18 """
19
20
21 import smtplib
22 import logging
23 import re
24
25 from difflib import SequenceMatcher
26 from email.mime.text import MIMEText
27 from email.mime.multipart import MIMEMultipart
28 from os.path import isdir
29 from collections import OrderedDict, defaultdict
30
31 from pal_errors import PresentationError
32
33
34 class AlertingError(PresentationError):
35     """Exception(s) raised by the alerting module.
36
37     When raising this exception, put this information to the message in this
38     order:
39      - short description of the encountered problem (parameter msg),
40      - relevant messages if there are any collected, e.g., from caught
41        exception (optional parameter details),
42      - relevant data if there are any collected (optional parameter details).
43     """
44
45     def __init__(self, msg, details=u'', level=u"CRITICAL"):
46         """Sets the exception message and the level.
47
48         :param msg: Short description of the encountered problem.
49         :param details: Relevant messages if there are any collected, e.g.,
50             from caught exception (optional parameter details), or relevant data
51             if there are any collected (optional parameter details).
52         :param level: Level of the error, possible choices are: "DEBUG", "INFO",
53             "WARNING", "ERROR" and "CRITICAL".
54         :type msg: str
55         :type details: str
56         :type level: str
57         """
58
59         super(AlertingError, self).__init__(f"Alerting: {msg}", details, level)
60
61     def __repr__(self):
62         return (
63             f"AlertingError(msg={self._msg!r},details={self._details!r},"
64             f"level={self._level!r})"
65         )
66
67
68 class Alerting:
69     """Class implementing the alerting mechanism.
70     """
71
72     def __init__(self, spec):
73         """Initialization.
74
75         :param spec: The CPTA specification.
76         :type spec: Specification
77         """
78
79         # Implemented alerts:
80         self._implemented_alerts = (u"failed-tests", )
81
82         self._spec = spec
83
84         self.error_msgs = list()
85
86         try:
87             self._spec_alert = spec.alerting
88         except KeyError as err:
89             raise AlertingError(
90                 u"Alerting is not configured, skipped.", repr(err), u"WARNING"
91             )
92
93         self._path_failed_tests = spec.environment[u"paths"][u"DIR[STATIC,VPP]"]
94
95         # Verify and validate input specification:
96         self.configs = self._spec_alert.get(u"configurations", None)
97         if not self.configs:
98             raise AlertingError(u"No alert configuration is specified.")
99         for config_type, config_data in self.configs.items():
100             if config_type == u"email":
101                 if not config_data.get(u"server", None):
102                     raise AlertingError(u"Parameter 'server' is missing.")
103                 if not config_data.get(u"address-to", None):
104                     raise AlertingError(u"Parameter 'address-to' (recipient) "
105                                         u"is missing.")
106                 if not config_data.get(u"address-from", None):
107                     raise AlertingError(u"Parameter 'address-from' (sender) is "
108                                         u"missing.")
109             elif config_type == u"jenkins":
110                 if not isdir(config_data.get(u"output-dir", u"")):
111                     raise AlertingError(u"Parameter 'output-dir' is "
112                                         u"missing or it is not a directory.")
113                 if not config_data.get(u"output-file", None):
114                     raise AlertingError(u"Parameter 'output-file' is missing.")
115             else:
116                 raise AlertingError(
117                     f"Alert of type {config_type} is not implemented."
118                 )
119
120         self.alerts = self._spec_alert.get(u"alerts", None)
121         if not self.alerts:
122             raise AlertingError(u"No alert is specified.")
123         for alert_data in self.alerts.values():
124             if not alert_data.get(u"title", None):
125                 raise AlertingError(u"Parameter 'title' is missing.")
126             if not alert_data.get(u"type", None) in self._implemented_alerts:
127                 raise AlertingError(u"Parameter 'failed-tests' is missing or "
128                                     u"incorrect.")
129             if not alert_data.get(u"way", None) in self.configs.keys():
130                 raise AlertingError(u"Parameter 'way' is missing or incorrect.")
131             if not alert_data.get(u"include", None):
132                 raise AlertingError(u"Parameter 'include' is missing or the "
133                                     u"list is empty.")
134
135     def __str__(self):
136         """Return string with human readable description of the alert.
137
138         :returns: Readable description.
139         :rtype: str
140         """
141         return f"configs={self.configs}, alerts={self.alerts}"
142
143     def __repr__(self):
144         """Return string executable as Python constructor call.
145
146         :returns: Executable constructor call.
147         :rtype: str
148         """
149         return f"Alerting(spec={self._spec})"
150
151     def generate_alerts(self):
152         """Generate alert(s) using specified way(s).
153         """
154
155         for alert_data in self.alerts.values():
156             if alert_data[u"way"] == u"jenkins":
157                 self._generate_email_body(alert_data)
158             else:
159                 raise AlertingError(
160                     f"Alert with way {alert_data[u'way']} is not implemented."
161                 )
162
163     @staticmethod
164     def _send_email(server, addr_from, addr_to, subject, text=None, html=None):
165         """Send an email using predefined configuration.
166
167         :param server: SMTP server used to send email.
168         :param addr_from: Sender address.
169         :param addr_to: Recipient address(es).
170         :param subject: Subject of the email.
171         :param text: Message in the ASCII text format.
172         :param html: Message in the HTML format.
173         :type server: str
174         :type addr_from: str
175         :type addr_to: list
176         :type subject: str
177         :type text: str
178         :type html: str
179         """
180
181         if not text and not html:
182             raise AlertingError(u"No text/data to send.")
183
184         msg = MIMEMultipart(u'alternative')
185         msg[u'Subject'] = subject
186         msg[u'From'] = addr_from
187         msg[u'To'] = u", ".join(addr_to)
188
189         if text:
190             msg.attach(MIMEText(text, u'plain'))
191         if html:
192             msg.attach(MIMEText(html, u'html'))
193
194         smtp_server = None
195         try:
196             logging.info(f"Trying to send alert {subject} ...")
197             logging.debug(f"SMTP Server: {server}")
198             logging.debug(f"From: {addr_from}")
199             logging.debug(f"To: {u', '.join(addr_to)}")
200             logging.debug(f"Message: {msg.as_string()}")
201             smtp_server = smtplib.SMTP(server)
202             smtp_server.sendmail(addr_from, addr_to, msg.as_string())
203         except smtplib.SMTPException as err:
204             raise AlertingError(u"Not possible to send the alert via email.",
205                                 str(err))
206         finally:
207             if smtp_server:
208                 smtp_server.quit()
209
210     def _get_compressed_failed_tests(self, alert, test_set, sort=True):
211         """Return the dictionary with compressed faild tests. The compression is
212         done by grouping the tests from the same area but with different NICs,
213         frame sizes and number of processor cores.
214
215         For example, the failed tests:
216           10ge2p1x520-64b-1c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
217           10ge2p1x520-64b-2c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
218           10ge2p1x520-64b-4c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
219           10ge2p1x520-imix-1c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
220           10ge2p1x520-imix-2c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
221           10ge2p1x520-imix-4c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
222
223         will be represented as:
224           ethip4udp-ip4scale4000-udpsrcscale15-nat44 \
225           (10ge2p1x520, 64b, imix, 1c, 2c, 4c)
226
227         Structure of returned data:
228
229         {
230             "trimmed_TC_name_1": {
231                 "nics": [],
232                 "framesizes": [],
233                 "cores": []
234             }
235             ...
236             "trimmed_TC_name_N": {
237                 "nics": [],
238                 "framesizes": [],
239                 "cores": []
240             }
241         }
242
243         :param alert: Files are created for this alert.
244         :param test_set: Specifies which set of tests will be included in the
245             result. Its name is the same as the name of file with failed tests.
246         :param sort: If True, the failed tests are sorted alphabetically.
247         :type alert: dict
248         :type test_set: str
249         :type sort: bool
250         :returns: CSIT build number, VPP version, Number of passed tests,
251             Number of failed tests, Compressed failed tests.
252         :rtype: tuple(str, str, int, int, str, OrderedDict)
253         """
254
255         directory = self.configs[alert[u"way"]][u"output-dir"]
256         failed_tests = defaultdict(dict)
257         file_path = f"{directory}/{test_set}.txt"
258         version = u""
259         try:
260             with open(file_path, u'r') as f_txt:
261                 for idx, line in enumerate(f_txt):
262                     if idx == 0:
263                         build = line[:-1]
264                         continue
265                     if idx == 1:
266                         version = line[:-1]
267                         continue
268                     if idx == 2:
269                         passed = line[:-1]
270                         continue
271                     if idx == 3:
272                         failed = line[:-1]
273                         continue
274                     if idx == 4:
275                         minutes = int(line[:-1]) // 60000
276                         duration = f"{(minutes // 60):02d}:{(minutes % 60):02d}"
277                         continue
278                     try:
279                         line, error_msg = line[:-1].split(u'###', maxsplit=1)
280                         test = line.split(u'-')
281                         name = u'-'.join(test[3:-1])
282                         if len(error_msg) > 128:
283                             if u";" in error_msg[128:256]:
284                                 error_msg = \
285                                     f"{error_msg[:128]}" \
286                                     f"{error_msg[128:].split(u';', 1)[0]}..."
287                             elif u":" in error_msg[128:256]:
288                                 error_msg = \
289                                     f"{error_msg[:128]}" \
290                                     f"{error_msg[128:].split(u':', 1)[0]}..."
291                             elif u"." in error_msg[128:256]:
292                                 error_msg = \
293                                     f"{error_msg[:128]}" \
294                                     f"{error_msg[128:].split(u'.', 1)[0]}..."
295                             elif u"?" in error_msg[128:256]:
296                                 error_msg = \
297                                     f"{error_msg[:128]}" \
298                                     f"{error_msg[128:].split(u'?', 1)[0]}..."
299                             elif u"!" in error_msg[128:256]:
300                                 error_msg = \
301                                     f"{error_msg[:128]}" \
302                                     f"{error_msg[128:].split(u'!', 1)[0]}..."
303                             elif u"," in error_msg[128:256]:
304                                 error_msg = \
305                                     f"{error_msg[:128]}" \
306                                     f"{error_msg[128:].split(u',', 1)[0]}..."
307                             elif u" " in error_msg[128:256]:
308                                 error_msg = \
309                                     f"{error_msg[:128]}" \
310                                     f"{error_msg[128:].split(u' ', 1)[0]}..."
311                             else:
312                                 error_msg = error_msg[:128]
313
314                     except ValueError:
315                         continue
316
317                     for e_msg in self.error_msgs:
318                         if SequenceMatcher(None, e_msg,
319                                            error_msg).ratio() > 0.5:
320                             error_msg = e_msg
321                             break
322                     if error_msg not in self.error_msgs:
323                         self.error_msgs.append(error_msg)
324
325                     error_msg_index = self.error_msgs.index(error_msg)
326
327                     if failed_tests.get(name, {}).get(error_msg_index) is None:
328                         failed_tests[name][error_msg_index] = \
329                             dict(nics=list(),
330                                  framesizes=list(),
331                                  cores=list())
332
333                     if test[0] not in \
334                             failed_tests[name][error_msg_index][u"nics"]:
335                         failed_tests[name][error_msg_index][u"nics"].\
336                             append(test[0])
337                     if test[1] not in \
338                             failed_tests[name][error_msg_index][u"framesizes"]:
339                         failed_tests[name][error_msg_index][u"framesizes"].\
340                             append(test[1])
341                     check_core = test[2] + f"[{str(error_msg_index)}]"
342                     if check_core not in \
343                             failed_tests[name][error_msg_index][u"cores"]:
344                         failed_tests[name][error_msg_index][u"cores"].\
345                             append(test[2] + "[" + str(error_msg_index) + "]")
346
347         except IOError:
348             logging.error(f"No such file or directory: {file_path}")
349             return None, None, None, None, None, None
350         if sort:
351             sorted_failed_tests = OrderedDict()
352             for key in sorted(failed_tests.keys()):
353                 sorted_failed_tests[key] = failed_tests[key]
354             return build, version, passed, failed, duration, sorted_failed_tests
355
356         return build, version, passed, failed, duration, failed_tests
357
358     def _list_gressions(self, alert, idx, header, re_pro):
359         """Create a file with regressions or progressions for the test set
360         specified by idx.
361
362         :param alert: Files are created for this alert.
363         :param idx: Index of the test set as it is specified in the
364             specification file.
365         :param header: The header of the list of [re|pro]gressions.
366         :param re_pro: 'regressions' or 'progressions'.
367         :type alert: dict
368         :type idx: int
369         :type header: str
370         :type re_pro: str
371         """
372
373         if re_pro not in (u"regressions", u"progressions"):
374             return
375
376         in_file = (
377             f"{self.configs[alert[u'way']][u'output-dir']}/"
378             f"{re_pro}-{alert[u'urls'][idx].split(u'/')[-1]}.txt"
379         )
380         out_file = (
381             f"{self.configs[alert[u'way']][u'output-dir']}/"
382             f"trending-{re_pro}.txt"
383         )
384
385         try:
386             with open(in_file, u'r') as txt_file:
387                 file_content = txt_file.read()
388                 with open(out_file, u'a+') as reg_file:
389                     reg_file.write(header)
390                     if file_content:
391                         reg_file.write(file_content)
392                     else:
393                         reg_file.write(f"No {re_pro}")
394         except IOError as err:
395             logging.warning(repr(err))
396
397     def _generate_email_body(self, alert):
398         """Create the file which is used in the generated alert.
399
400         :param alert: Files are created for this alert.
401         :type alert: dict
402         """
403
404         if alert[u"type"] != u"failed-tests":
405             raise AlertingError(
406                 f"Alert of type {alert[u'type']} is not implemented."
407             )
408
409         text = u""
410
411         legend = (f"Legend: Test-name  NIC  Frame-size  Trend[Mpps]  Runs[#]  "
412                   f"Long-Term change[%]")
413
414         out_file = (
415             f"{self.configs[alert[u'way']][u'output-dir']}/"
416             f"trending-regressions.txt"
417         )
418         try:
419             with open(out_file, u'w') as reg_file:
420                 reg_file.write(legend)
421         except IOError:
422             logging.error(f"Not possible to write the file {out_file}.txt.")
423
424         out_file = (
425             f"{self.configs[alert[u'way']][u'output-dir']}/"
426             f"trending-progressions.txt"
427         )
428         try:
429             with open(out_file, u'w') as reg_file:
430                 reg_file.write(legend)
431         except IOError:
432             logging.error(f"Not possible to write the file {out_file}.txt.")
433
434         for idx, test_set in enumerate(alert.get(u"include", list())):
435             test_set_short = u""
436             device = u""
437             try:
438                 groups = re.search(
439                     re.compile(
440                         r'((vpp|dpdk)-\dn-(skx|clx|tsh|dnv|zn2|tx2|icx)-.*)'
441                     ),
442                     test_set
443                 )
444                 test_set_short = groups.group(1)
445                 device = groups.group(2)
446             except (AttributeError, IndexError):
447                 logging.error(
448                     f"The test set {test_set} does not include information "
449                     f"about test bed. Using empty string instead."
450                 )
451             build, version, passed, failed, duration, failed_tests = \
452                 self._get_compressed_failed_tests(alert, test_set)
453             if build is None:
454                 text += (
455                     f"\n\nNo input data available for {test_set_short}. "
456                     f"See CSIT job {alert[u'urls'][idx]} for more "
457                     f"information.\n"
458                 )
459                 continue
460             text += (
461                 f"\n\n{test_set_short}, "
462                 f"{failed} tests failed, "
463                 f"{passed} tests passed, "
464                 f"duration: {duration}, "
465                 f"CSIT build: {alert[u'urls'][idx]}/{build}, "
466                 f"{device} version: {version}\n\n"
467             )
468
469             class MaxLens():
470                 """Class to store the max lengths of strings displayed in
471                 failed tests list.
472                 """
473                 def __init__(self, tst_name, nics, framesizes, cores):
474                     """Initialisation.
475
476                     :param tst_name: Name of the test.
477                     :param nics: NICs used in the test.
478                     :param framesizes: Frame sizes used in the tests
479                     :param cores: Cores used in th test.
480                     """
481                     self.name = tst_name
482                     self.nics = nics
483                     self.frmsizes = framesizes
484                     self.cores = cores
485
486             max_len = MaxLens(0, 0, 0, 0)
487
488             for test, message in failed_tests.items():
489                 for e_message, params in message.items():
490                     failed_tests[test][e_message][u"nics"] = \
491                         u" ".join(sorted(params[u"nics"]))
492                     failed_tests[test][e_message][u"framesizes"] = \
493                         u" ".join(sorted(params[u"framesizes"]))
494                     failed_tests[test][e_message][u"cores"] = \
495                         u" ".join(sorted(params[u"cores"]))
496                     if len(test) > max_len.name:
497                         max_len.name = len(test)
498                     if len(failed_tests[test][e_message][u"nics"]) > \
499                             max_len.nics:
500                         max_len.nics = \
501                             len(failed_tests[test][e_message][u"nics"])
502                     if len(failed_tests[test][e_message][u"framesizes"]) > \
503                             max_len.frmsizes:
504                         max_len.frmsizes = \
505                             len(failed_tests[test][e_message][u"framesizes"])
506                     if len(failed_tests[test][e_message][u"cores"]) > \
507                             max_len.cores:
508                         max_len.cores = \
509                             len(failed_tests[test][e_message][u"cores"])
510
511             for test, message in failed_tests.items():
512                 test_added = False
513                 for e_message, params in message.items():
514                     if not test_added:
515                         test_added = True
516                     else:
517                         test = ""
518                     text += (
519                         f"{test + u' ' * (max_len.name - len(test))}  "
520                         f"{params[u'nics']}"
521                         f"{u' ' * (max_len.nics - len(params[u'nics']))}  "
522                         f"{params[u'framesizes']}"
523                         f"""{u' ' * (max_len.frmsizes
524                                      - len(params[u'framesizes']))}  """
525                         f"{params[u'cores']}"
526                         f"{u' ' * (max_len.cores - len(params[u'cores']))}\n"
527                     )
528
529             gression_hdr = (
530                 f"\n\n{test_set_short}, "
531                 f"CSIT build: {alert[u'urls'][idx]}/{build}, "
532                 f"{device} version: {version}\n\n"
533             )
534             # Add list of regressions:
535             self._list_gressions(alert, idx, gression_hdr, u"regressions")
536
537             # Add list of progressions:
538             self._list_gressions(alert, idx, gression_hdr, u"progressions")
539
540         text += f"\nFor detailed information visit: {alert[u'url-details']}\n"
541         file_name = f"{self.configs[alert[u'way']][u'output-dir']}/" \
542                     f"{self.configs[alert[u'way']][u'output-file']}"
543         logging.info(f"Writing the file {file_name}.txt ...")
544
545         text += f"\n\nLegend:\n\n"
546
547         for e_msg in self.error_msgs:
548             text += f"[{self.error_msgs.index(e_msg)}] - {e_msg}\n"
549
550         try:
551             with open(f"{file_name}.txt", u'w') as txt_file:
552                 txt_file.write(text)
553         except IOError:
554             logging.error(f"Not possible to write the file {file_name}.txt.")