1 # Copyright (c) 2019 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Generator of alerts:
25 from email.mime.text import MIMEText
26 from email.mime.multipart import MIMEMultipart
27 from os.path import isdir
28 from collections import OrderedDict
30 from pal_utils import get_last_completed_build_number
31 from pal_errors import PresentationError
34 class AlertingError(PresentationError):
35 """Exception(s) raised by the alerting module.
37 When raising this exception, put this information to the message in this
39 - short description of the encountered problem (parameter msg),
40 - relevant messages if there are any collected, e.g., from caught
41 exception (optional parameter details),
42 - relevant data if there are any collected (optional parameter details).
45 def __init__(self, msg, details=u'', level=u"CRITICAL"):
46 """Sets the exception message and the level.
48 :param msg: Short description of the encountered problem.
49 :param details: Relevant messages if there are any collected, e.g.,
50 from caught exception (optional parameter details), or relevant data
51 if there are any collected (optional parameter details).
52 :param level: Level of the error, possible choices are: "DEBUG", "INFO",
53 "WARNING", "ERROR" and "CRITICAL".
59 super(AlertingError, self).__init__(f"Alerting: {msg}", details, level)
63 f"AlertingError(msg={self._msg!r},details={self._details!r},"
64 f"level={self._level!r})"
69 """Class implementing the alerting mechanism.
72 def __init__(self, spec):
75 :param spec: The CPTA specification.
76 :type spec: Specification
80 self._implemented_alerts = (u"failed-tests", )
85 self._spec_alert = spec.alerting
86 except KeyError as err:
87 raise AlertingError(u"Alerting is not configured, skipped.",
91 self._path_failed_tests = spec.environment[u"paths"][u"DIR[STATIC,VPP]"]
93 # Verify and validate input specification:
94 self.configs = self._spec_alert.get(u"configurations", None)
96 raise AlertingError(u"No alert configuration is specified.")
97 for config_type, config_data in self.configs.items():
98 if config_type == u"email":
99 if not config_data.get(u"server", None):
100 raise AlertingError(u"Parameter 'server' is missing.")
101 if not config_data.get(u"address-to", None):
102 raise AlertingError(u"Parameter 'address-to' (recipient) "
104 if not config_data.get(u"address-from", None):
105 raise AlertingError(u"Parameter 'address-from' (sender) is "
107 elif config_type == u"jenkins":
108 if not isdir(config_data.get(u"output-dir", u"")):
109 raise AlertingError(u"Parameter 'output-dir' is "
110 u"missing or it is not a directory.")
111 if not config_data.get(u"output-file", None):
112 raise AlertingError(u"Parameter 'output-file' is missing.")
115 f"Alert of type {config_type} is not implemented."
118 self.alerts = self._spec_alert.get(u"alerts", None)
120 raise AlertingError(u"No alert is specified.")
121 for alert_data in self.alerts.values():
122 if not alert_data.get(u"title", None):
123 raise AlertingError(u"Parameter 'title' is missing.")
124 if not alert_data.get(u"type", None) in self._implemented_alerts:
125 raise AlertingError(u"Parameter 'failed-tests' is missing or "
127 if not alert_data.get(u"way", None) in self.configs.keys():
128 raise AlertingError(u"Parameter 'way' is missing or incorrect.")
129 if not alert_data.get(u"include", None):
130 raise AlertingError(u"Parameter 'include' is missing or the "
134 """Return string with human readable description of the alert.
136 :returns: Readable description.
139 return f"configs={self.configs}, alerts={self.alerts}"
142 """Return string executable as Python constructor call.
144 :returns: Executable constructor call.
147 return f"Alerting(spec={self._spec})"
149 def generate_alerts(self):
150 """Generate alert(s) using specified way(s).
153 for alert_data in self.alerts.values():
154 if alert_data[u"way"] == u"jenkins":
155 self._generate_email_body(alert_data)
158 f"Alert with way {alert_data[u'way']} is not implemented."
162 def _send_email(server, addr_from, addr_to, subject, text=None, html=None):
163 """Send an email using predefined configuration.
165 :param server: SMTP server used to send email.
166 :param addr_from: Sender address.
167 :param addr_to: Recipient address(es).
168 :param subject: Subject of the email.
169 :param text: Message in the ASCII text format.
170 :param html: Message in the HTML format.
179 if not text and not html:
180 raise AlertingError(u"No text/data to send.")
182 msg = MIMEMultipart(u'alternative')
183 msg[u'Subject'] = subject
184 msg[u'From'] = addr_from
185 msg[u'To'] = u", ".join(addr_to)
188 msg.attach(MIMEText(text, u'plain'))
190 msg.attach(MIMEText(html, u'html'))
194 logging.info(f"Trying to send alert {subject} ...")
195 logging.debug(f"SMTP Server: {server}")
196 logging.debug(f"From: {addr_from}")
197 logging.debug(f"To: {u', '.join(addr_to)}")
198 logging.debug(f"Message: {msg.as_string()}")
199 smtp_server = smtplib.SMTP(server)
200 smtp_server.sendmail(addr_from, addr_to, msg.as_string())
201 except smtplib.SMTPException as err:
202 raise AlertingError(u"Not possible to send the alert via email.",
208 def _get_compressed_failed_tests(self, alert, test_set, sort=True):
209 """Return the dictionary with compressed faild tests. The compression is
210 done by grouping the tests from the same area but with different NICs,
211 frame sizes and number of processor cores.
213 For example, the failed tests:
214 10ge2p1x520-64b-1c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
215 10ge2p1x520-64b-2c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
216 10ge2p1x520-64b-4c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
217 10ge2p1x520-imix-1c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
218 10ge2p1x520-imix-2c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
219 10ge2p1x520-imix-4c-ethip4udp-ip4scale4000-udpsrcscale15-nat44-mrr
221 will be represented as:
222 ethip4udp-ip4scale4000-udpsrcscale15-nat44 \
223 (10ge2p1x520, 64b, imix, 1c, 2c, 4c)
225 Structure of returned data:
228 "trimmed_TC_name_1": {
234 "trimmed_TC_name_N": {
241 :param alert: Files are created for this alert.
242 :param test_set: Specifies which set of tests will be included in the
243 result. Its name is the same as the name of file with failed tests.
244 :param sort: If True, the failed tests are sorted alphabetically.
248 :returns: CSIT build number, VPP version, Number of passed tests,
249 Number of failed tests, Compressed failed tests.
250 :rtype: tuple(str, str, int, int, OrderedDict)
253 directory = self.configs[alert[u"way"]][u"output-dir"]
254 failed_tests = OrderedDict()
255 file_path = f"{directory}/{test_set}.txt"
258 with open(file_path, u'r') as f_txt:
259 for idx, line in enumerate(f_txt):
273 test = line[:-1].split(u'-')
274 name = u'-'.join(test[3:-1])
277 if failed_tests.get(name, None) is None:
278 failed_tests[name] = dict(nics=list(),
281 if test[0] not in failed_tests[name][u"nics"]:
282 failed_tests[name][u"nics"].append(test[0])
283 if test[1] not in failed_tests[name][u"framesizes"]:
284 failed_tests[name][u"framesizes"].append(test[1])
285 if test[2] not in failed_tests[name][u"cores"]:
286 failed_tests[name][u"cores"].append(test[2])
288 logging.error(f"No such file or directory: {file_path}")
289 return None, None, None, None, None
291 sorted_failed_tests = OrderedDict()
292 for key in sorted(failed_tests.keys()):
293 sorted_failed_tests[key] = failed_tests[key]
294 return build, version, passed, failed, sorted_failed_tests
296 return build, version, passed, failed, failed_tests
298 def _list_gressions(self, alert, idx, header, re_pro):
299 """Create a file with regressions or progressions for the test set
302 :param alert: Files are created for this alert.
303 :param idx: Index of the test set as it is specified in the
305 :param header: The header of the list of [re|pro]gressions.
306 :param re_pro: 'regression' or 'progression'.
313 if re_pro not in (u"regressions", u"progressions"):
317 f"{self.configs[alert[u'way']][u'output-dir']}/"
318 f"{re_pro}-{alert[u'urls'][idx].split(u'/')[-1]}.txt"
321 f"{self.configs[alert[u'way']][u'output-dir']}/"
322 f"trending-{re_pro}.txt"
326 with open(in_file, u'r') as txt_file:
327 file_content = txt_file.read()
328 with open(out_file, u'a+') as reg_file:
329 reg_file.write(header)
331 reg_file.write(file_content)
333 reg_file.write(f"No {re_pro}")
334 except IOError as err:
335 logging.warning(repr(err))
337 def _generate_email_body(self, alert):
338 """Create the file which is used in the generated alert.
340 :param alert: Files are created for this alert.
344 if alert[u"type"] != u"failed-tests":
346 f"Alert of type {alert[u'type']} is not implemented."
350 for idx, test_set in enumerate(alert.get(u"include", list())):
356 r'((vpp|dpdk)-\dn-(skx|clx|hsw|tsh|dnv|zn2|tx2)-.*)'
360 test_set_short = groups.group(1)
361 device = groups.group(2)
362 except (AttributeError, IndexError):
364 f"The test set {test_set} does not include information "
365 f"about test bed. Using empty string instead."
367 build, version, passed, failed, failed_tests = \
368 self._get_compressed_failed_tests(alert, test_set)
371 f"\n\nNo input data available for {test_set_short}. "
372 f"See CSIT job {alert[u'urls'][idx]} for more "
377 f"\n\n{test_set_short}, {failed} tests failed, {passed} tests "
378 f"passed, CSIT build: {alert[u'urls'][idx]}/{build}, "
379 f"{device} version: {version}\n\n"
383 """Class to store the max lengths of strings displayed in
386 def __init__(self, tst_name, nics, framesizes, cores):
389 :param tst_name: Name of the test.
390 :param nics: NICs used in the test.
391 :param framesizes: Frame sizes used in the tests
392 :param cores: Cores used in th test.
396 self.frmsizes = framesizes
399 max_len = MaxLens(0, 0, 0, 0)
401 for name, params in failed_tests.items():
402 failed_tests[name][u"nics"] = u",".join(sorted(params[u"nics"]))
403 failed_tests[name][u"framesizes"] = \
404 u",".join(sorted(params[u"framesizes"]))
405 failed_tests[name][u"cores"] = \
406 u",".join(sorted(params[u"cores"]))
407 if len(name) > max_len.name:
408 max_len.name = len(name)
409 if len(failed_tests[name][u"nics"]) > max_len.nics:
410 max_len.nics = len(failed_tests[name][u"nics"])
411 if len(failed_tests[name][u"framesizes"]) > max_len.frmsizes:
412 max_len.frmsizes = len(failed_tests[name][u"framesizes"])
413 if len(failed_tests[name][u"cores"]) > max_len.cores:
414 max_len.cores = len(failed_tests[name][u"cores"])
416 for name, params in failed_tests.items():
418 f"{name + u' ' * (max_len.name - len(name))} "
420 f"{u' ' * (max_len.nics - len(params[u'nics']))} "
421 f"{params[u'framesizes']}"
422 f"{u' ' * (max_len.frmsizes-len(params[u'framesizes']))} "
423 f"{params[u'cores']}"
424 f"{u' ' * (max_len.cores - len(params[u'cores']))}\n"
428 f"\n\n{test_set_short}, "
429 f"CSIT build: {alert[u'urls'][idx]}/{build}, "
430 f"{device} version: {version}\n\n"
432 # Add list of regressions:
433 self._list_gressions(alert, idx, gression_hdr, u"regressions")
435 # Add list of progressions:
436 self._list_gressions(alert, idx, gression_hdr, u"progressions")
438 text += f"\nFor detailed information visit: {alert[u'url-details']}\n"
439 file_name = f"{self.configs[alert[u'way']][u'output-dir']}/" \
440 f"{self.configs[alert[u'way']][u'output-file']}"
441 logging.info(f"Writing the file {file_name}.txt ...")
444 with open(f"{file_name}.txt", u'w') as txt_file:
447 logging.error(f"Not possible to write the file {file_name}.txt.")