1 # Copyright (c) 2019 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Algorithms to generate tables.
22 from string import replace
23 from collections import OrderedDict
24 from numpy import nan, isnan
25 from xml.etree import ElementTree as ET
26 from datetime import datetime as dt
27 from datetime import timedelta
29 from utils import mean, stdev, relative_change, classify_anomalies, \
30 convert_csv_to_pretty_txt, relative_change_stdev
33 REGEX_NIC = re.compile(r'\d*ge\dp\d\D*\d*')
36 def generate_tables(spec, data):
37 """Generate all tables specified in the specification file.
39 :param spec: Specification read from the specification file.
40 :param data: Data to process.
41 :type spec: Specification
45 logging.info("Generating the tables ...")
46 for table in spec.tables:
48 eval(table["algorithm"])(table, data)
49 except NameError as err:
50 logging.error("Probably algorithm '{alg}' is not defined: {err}".
51 format(alg=table["algorithm"], err=repr(err)))
55 def table_details(table, input_data):
56 """Generate the table(s) with algorithm: table_detailed_test_results
57 specified in the specification file.
59 :param table: Table to generate.
60 :param input_data: Data to process.
61 :type table: pandas.Series
62 :type input_data: InputData
65 logging.info(" Generating the table {0} ...".
66 format(table.get("title", "")))
69 logging.info(" Creating the data set for the {0} '{1}'.".
70 format(table.get("type", ""), table.get("title", "")))
71 data = input_data.filter_data(table)
73 # Prepare the header of the tables
75 for column in table["columns"]:
76 header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
78 # Generate the data for the table according to the model in the table
80 job = table["data"].keys()[0]
81 build = str(table["data"][job][0])
83 suites = input_data.suites(job, build)
85 logging.error(" No data available. The table will not be generated.")
88 for suite_longname, suite in suites.iteritems():
90 suite_name = suite["name"]
92 for test in data[job][build].keys():
93 if data[job][build][test]["parent"] in suite_name:
95 for column in table["columns"]:
97 col_data = str(data[job][build][test][column["data"].
98 split(" ")[1]]).replace('"', '""')
99 if column["data"].split(" ")[1] in ("conf-history",
101 col_data = replace(col_data, " |br| ", "",
103 col_data = " |prein| {0} |preout| ".\
104 format(col_data[:-5])
105 row_lst.append('"{0}"'.format(col_data))
107 row_lst.append("No data")
108 table_lst.append(row_lst)
110 # Write the data to file
112 file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
113 table["output-file-ext"])
114 logging.info(" Writing file: '{}'".format(file_name))
115 with open(file_name, "w") as file_handler:
116 file_handler.write(",".join(header) + "\n")
117 for item in table_lst:
118 file_handler.write(",".join(item) + "\n")
120 logging.info(" Done.")
123 def table_merged_details(table, input_data):
124 """Generate the table(s) with algorithm: table_merged_details
125 specified in the specification file.
127 :param table: Table to generate.
128 :param input_data: Data to process.
129 :type table: pandas.Series
130 :type input_data: InputData
133 logging.info(" Generating the table {0} ...".
134 format(table.get("title", "")))
137 logging.info(" Creating the data set for the {0} '{1}'.".
138 format(table.get("type", ""), table.get("title", "")))
139 data = input_data.filter_data(table)
140 data = input_data.merge_data(data)
141 data.sort_index(inplace=True)
143 logging.info(" Creating the data set for the {0} '{1}'.".
144 format(table.get("type", ""), table.get("title", "")))
145 suites = input_data.filter_data(table, data_set="suites")
146 suites = input_data.merge_data(suites)
148 # Prepare the header of the tables
150 for column in table["columns"]:
151 header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
153 for _, suite in suites.iteritems():
155 suite_name = suite["name"]
157 for test in data.keys():
158 if data[test]["parent"] in suite_name:
160 for column in table["columns"]:
162 col_data = str(data[test][column["data"].
163 split(" ")[1]]).replace('"', '""')
164 col_data = replace(col_data, "No Data",
166 if column["data"].split(" ")[1] in ("conf-history",
168 col_data = replace(col_data, " |br| ", "",
170 col_data = " |prein| {0} |preout| ".\
171 format(col_data[:-5])
172 row_lst.append('"{0}"'.format(col_data))
174 row_lst.append('"Not captured"')
175 table_lst.append(row_lst)
177 # Write the data to file
179 file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
180 table["output-file-ext"])
181 logging.info(" Writing file: '{}'".format(file_name))
182 with open(file_name, "w") as file_handler:
183 file_handler.write(",".join(header) + "\n")
184 for item in table_lst:
185 file_handler.write(",".join(item) + "\n")
187 logging.info(" Done.")
190 def table_performance_comparison(table, input_data):
191 """Generate the table(s) with algorithm: table_performance_comparison
192 specified in the specification file.
194 :param table: Table to generate.
195 :param input_data: Data to process.
196 :type table: pandas.Series
197 :type input_data: InputData
200 logging.info(" Generating the table {0} ...".
201 format(table.get("title", "")))
204 logging.info(" Creating the data set for the {0} '{1}'.".
205 format(table.get("type", ""), table.get("title", "")))
206 data = input_data.filter_data(table, continue_on_error=True)
208 # Prepare the header of the tables
210 header = ["Test case", ]
212 if table["include-tests"] == "MRR":
213 hdr_param = "Receive Rate"
215 hdr_param = "Throughput"
217 history = table.get("history", None)
221 ["{0} {1} [Mpps]".format(item["title"], hdr_param),
222 "{0} Stdev [Mpps]".format(item["title"])])
224 ["{0} {1} [Mpps]".format(table["reference"]["title"], hdr_param),
225 "{0} Stdev [Mpps]".format(table["reference"]["title"]),
226 "{0} {1} [Mpps]".format(table["compare"]["title"], hdr_param),
227 "{0} Stdev [Mpps]".format(table["compare"]["title"]),
229 header_str = ",".join(header) + "\n"
230 except (AttributeError, KeyError) as err:
231 logging.error("The model is invalid, missing parameter: {0}".
235 # Prepare data to the table:
237 for job, builds in table["reference"]["data"].items():
239 for tst_name, tst_data in data[job][str(build)].iteritems():
240 tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\
241 replace("-ndrpdr", "").replace("-pdrdisc", "").\
242 replace("-ndrdisc", "").replace("-pdr", "").\
243 replace("-ndr", "").\
244 replace("1t1c", "1c").replace("2t1c", "1c").\
245 replace("2t2c", "2c").replace("4t2c", "2c").\
246 replace("4t4c", "4c").replace("8t4c", "4c")
247 if "across topologies" in table["title"].lower():
248 tst_name_mod = tst_name_mod.replace("2n1l-", "")
249 if tbl_dict.get(tst_name_mod, None) is None:
250 groups = re.search(REGEX_NIC, tst_data["parent"])
251 nic = groups.group(0) if groups else ""
252 name = "{0}-{1}".format(nic, "-".join(tst_data["name"].
254 if "across testbeds" in table["title"].lower() or \
255 "across topologies" in table["title"].lower():
257 replace("1t1c", "1c").replace("2t1c", "1c").\
258 replace("2t2c", "2c").replace("4t2c", "2c").\
259 replace("4t4c", "4c").replace("8t4c", "4c")
260 tbl_dict[tst_name_mod] = {"name": name,
264 # TODO: Re-work when NDRPDRDISC tests are not used
265 if table["include-tests"] == "MRR":
266 tbl_dict[tst_name_mod]["ref-data"]. \
267 append(tst_data["result"]["receive-rate"].avg)
268 elif table["include-tests"] == "PDR":
269 if tst_data["type"] == "PDR":
270 tbl_dict[tst_name_mod]["ref-data"]. \
271 append(tst_data["throughput"]["value"])
272 elif tst_data["type"] == "NDRPDR":
273 tbl_dict[tst_name_mod]["ref-data"].append(
274 tst_data["throughput"]["PDR"]["LOWER"])
275 elif table["include-tests"] == "NDR":
276 if tst_data["type"] == "NDR":
277 tbl_dict[tst_name_mod]["ref-data"]. \
278 append(tst_data["throughput"]["value"])
279 elif tst_data["type"] == "NDRPDR":
280 tbl_dict[tst_name_mod]["ref-data"].append(
281 tst_data["throughput"]["NDR"]["LOWER"])
285 pass # No data in output.xml for this test
287 for job, builds in table["compare"]["data"].items():
289 for tst_name, tst_data in data[job][str(build)].iteritems():
290 tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \
291 replace("-ndrpdr", "").replace("-pdrdisc", ""). \
292 replace("-ndrdisc", "").replace("-pdr", ""). \
293 replace("-ndr", "").\
294 replace("1t1c", "1c").replace("2t1c", "1c").\
295 replace("2t2c", "2c").replace("4t2c", "2c").\
296 replace("4t4c", "4c").replace("8t4c", "4c")
297 if "across topologies" in table["title"].lower():
298 tst_name_mod = tst_name_mod.replace("2n1l-", "")
300 # TODO: Re-work when NDRPDRDISC tests are not used
301 if table["include-tests"] == "MRR":
302 tbl_dict[tst_name_mod]["cmp-data"]. \
303 append(tst_data["result"]["receive-rate"].avg)
304 elif table["include-tests"] == "PDR":
305 if tst_data["type"] == "PDR":
306 tbl_dict[tst_name_mod]["cmp-data"]. \
307 append(tst_data["throughput"]["value"])
308 elif tst_data["type"] == "NDRPDR":
309 tbl_dict[tst_name_mod]["cmp-data"].append(
310 tst_data["throughput"]["PDR"]["LOWER"])
311 elif table["include-tests"] == "NDR":
312 if tst_data["type"] == "NDR":
313 tbl_dict[tst_name_mod]["cmp-data"]. \
314 append(tst_data["throughput"]["value"])
315 elif tst_data["type"] == "NDRPDR":
316 tbl_dict[tst_name_mod]["cmp-data"].append(
317 tst_data["throughput"]["NDR"]["LOWER"])
323 tbl_dict.pop(tst_name_mod, None)
326 for job, builds in item["data"].items():
328 for tst_name, tst_data in data[job][str(build)].iteritems():
329 tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \
330 replace("-ndrpdr", "").replace("-pdrdisc", ""). \
331 replace("-ndrdisc", "").replace("-pdr", ""). \
332 replace("-ndr", "").\
333 replace("1t1c", "1c").replace("2t1c", "1c").\
334 replace("2t2c", "2c").replace("4t2c", "2c").\
335 replace("4t4c", "4c").replace("8t4c", "4c")
336 if "across topologies" in table["title"].lower():
337 tst_name_mod = tst_name_mod.replace("2n1l-", "")
338 if tbl_dict.get(tst_name_mod, None) is None:
340 if tbl_dict[tst_name_mod].get("history", None) is None:
341 tbl_dict[tst_name_mod]["history"] = OrderedDict()
342 if tbl_dict[tst_name_mod]["history"].get(item["title"],
344 tbl_dict[tst_name_mod]["history"][item["title"]] = \
347 # TODO: Re-work when NDRPDRDISC tests are not used
348 if table["include-tests"] == "MRR":
349 tbl_dict[tst_name_mod]["history"][item["title"
350 ]].append(tst_data["result"]["receive-rate"].
352 elif table["include-tests"] == "PDR":
353 if tst_data["type"] == "PDR":
354 tbl_dict[tst_name_mod]["history"][
356 append(tst_data["throughput"]["value"])
357 elif tst_data["type"] == "NDRPDR":
358 tbl_dict[tst_name_mod]["history"][item[
359 "title"]].append(tst_data["throughput"][
361 elif table["include-tests"] == "NDR":
362 if tst_data["type"] == "NDR":
363 tbl_dict[tst_name_mod]["history"][
365 append(tst_data["throughput"]["value"])
366 elif tst_data["type"] == "NDRPDR":
367 tbl_dict[tst_name_mod]["history"][item[
368 "title"]].append(tst_data["throughput"][
372 except (TypeError, KeyError):
376 for tst_name in tbl_dict.keys():
377 item = [tbl_dict[tst_name]["name"], ]
379 if tbl_dict[tst_name].get("history", None) is not None:
380 for hist_data in tbl_dict[tst_name]["history"].values():
382 item.append(round(mean(hist_data) / 1000000, 2))
383 item.append(round(stdev(hist_data) / 1000000, 2))
385 item.extend([None, None])
387 item.extend([None, None])
388 data_t = tbl_dict[tst_name]["ref-data"]
390 item.append(round(mean(data_t) / 1000000, 2))
391 item.append(round(stdev(data_t) / 1000000, 2))
393 item.extend([None, None])
394 data_t = tbl_dict[tst_name]["cmp-data"]
396 item.append(round(mean(data_t) / 1000000, 2))
397 item.append(round(stdev(data_t) / 1000000, 2))
399 item.extend([None, None])
400 if item[-4] is not None and item[-2] is not None and item[-4] != 0:
401 item.append(int(relative_change(float(item[-4]), float(item[-2]))))
402 if len(item) == len(header):
405 # Sort the table according to the relative change
406 tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
408 # Generate csv tables:
409 csv_file = "{0}.csv".format(table["output-file"])
410 with open(csv_file, "w") as file_handler:
411 file_handler.write(header_str)
413 file_handler.write(",".join([str(item) for item in test]) + "\n")
415 convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
418 def table_nics_comparison(table, input_data):
419 """Generate the table(s) with algorithm: table_nics_comparison
420 specified in the specification file.
422 :param table: Table to generate.
423 :param input_data: Data to process.
424 :type table: pandas.Series
425 :type input_data: InputData
428 logging.info(" Generating the table {0} ...".
429 format(table.get("title", "")))
432 logging.info(" Creating the data set for the {0} '{1}'.".
433 format(table.get("type", ""), table.get("title", "")))
434 data = input_data.filter_data(table, continue_on_error=True)
436 # Prepare the header of the tables
438 header = ["Test case", ]
440 if table["include-tests"] == "MRR":
441 hdr_param = "Receive Rate"
443 hdr_param = "Throughput"
446 ["{0} {1} [Mpps]".format(table["reference"]["title"], hdr_param),
447 "{0} Stdev [Mpps]".format(table["reference"]["title"]),
448 "{0} {1} [Mpps]".format(table["compare"]["title"], hdr_param),
449 "{0} Stdev [Mpps]".format(table["compare"]["title"]),
451 header_str = ",".join(header) + "\n"
452 except (AttributeError, KeyError) as err:
453 logging.error("The model is invalid, missing parameter: {0}".
457 # Prepare data to the table:
459 for job, builds in table["data"].items():
461 for tst_name, tst_data in data[job][str(build)].iteritems():
462 tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\
463 replace("-ndrpdr", "").replace("-pdrdisc", "").\
464 replace("-ndrdisc", "").replace("-pdr", "").\
465 replace("-ndr", "").\
466 replace("1t1c", "1c").replace("2t1c", "1c").\
467 replace("2t2c", "2c").replace("4t2c", "2c").\
468 replace("4t4c", "4c").replace("8t4c", "4c")
469 tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod)
470 if tbl_dict.get(tst_name_mod, None) is None:
471 name = "-".join(tst_data["name"].split("-")[:-1])
472 tbl_dict[tst_name_mod] = {"name": name,
476 if table["include-tests"] == "MRR":
477 result = tst_data["result"]["receive-rate"].avg
478 elif table["include-tests"] == "PDR":
479 result = tst_data["throughput"]["PDR"]["LOWER"]
480 elif table["include-tests"] == "NDR":
481 result = tst_data["throughput"]["NDR"]["LOWER"]
486 if table["reference"]["nic"] in tst_data["tags"]:
487 tbl_dict[tst_name_mod]["ref-data"].append(result)
488 elif table["compare"]["nic"] in tst_data["tags"]:
489 tbl_dict[tst_name_mod]["cmp-data"].append(result)
490 except (TypeError, KeyError) as err:
491 logging.debug("No data for {0}".format(tst_name))
492 logging.debug(repr(err))
493 # No data in output.xml for this test
496 for tst_name in tbl_dict.keys():
497 item = [tbl_dict[tst_name]["name"], ]
498 data_t = tbl_dict[tst_name]["ref-data"]
500 item.append(round(mean(data_t) / 1000000, 2))
501 item.append(round(stdev(data_t) / 1000000, 2))
503 item.extend([None, None])
504 data_t = tbl_dict[tst_name]["cmp-data"]
506 item.append(round(mean(data_t) / 1000000, 2))
507 item.append(round(stdev(data_t) / 1000000, 2))
509 item.extend([None, None])
510 if item[-4] is not None and item[-2] is not None and item[-4] != 0:
511 item.append(int(relative_change(float(item[-4]), float(item[-2]))))
512 if len(item) == len(header):
515 # Sort the table according to the relative change
516 tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
518 # Generate csv tables:
519 csv_file = "{0}.csv".format(table["output-file"])
520 with open(csv_file, "w") as file_handler:
521 file_handler.write(header_str)
523 file_handler.write(",".join([str(item) for item in test]) + "\n")
525 convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
528 def table_soak_vs_ndr(table, input_data):
529 """Generate the table(s) with algorithm: table_soak_vs_ndr
530 specified in the specification file.
532 :param table: Table to generate.
533 :param input_data: Data to process.
534 :type table: pandas.Series
535 :type input_data: InputData
538 logging.info(" Generating the table {0} ...".
539 format(table.get("title", "")))
542 logging.info(" Creating the data set for the {0} '{1}'.".
543 format(table.get("type", ""), table.get("title", "")))
544 data = input_data.filter_data(table, continue_on_error=True)
546 # Prepare the header of the table
550 "{0} Throughput [Mpps]".format(table["reference"]["title"]),
551 "{0} Stdev [Mpps]".format(table["reference"]["title"]),
552 "{0} Throughput [Mpps]".format(table["compare"]["title"]),
553 "{0} Stdev [Mpps]".format(table["compare"]["title"]),
554 "Delta [%]", "Stdev of delta [%]"]
555 header_str = ",".join(header) + "\n"
556 except (AttributeError, KeyError) as err:
557 logging.error("The model is invalid, missing parameter: {0}".
561 # Create a list of available SOAK test results:
563 for job, builds in table["compare"]["data"].items():
565 for tst_name, tst_data in data[job][str(build)].iteritems():
566 if tst_data["type"] == "SOAK":
567 tst_name_mod = tst_name.replace("-soak", "")
568 if tbl_dict.get(tst_name_mod, None) is None:
569 groups = re.search(REGEX_NIC, tst_data["parent"])
570 nic = groups.group(0) if groups else ""
571 name = "{0}-{1}".format(nic, "-".join(tst_data["name"].
573 tbl_dict[tst_name_mod] = {
579 tbl_dict[tst_name_mod]["cmp-data"].append(
580 tst_data["throughput"]["LOWER"])
581 except (KeyError, TypeError):
583 tests_lst = tbl_dict.keys()
585 # Add corresponding NDR test results:
586 for job, builds in table["reference"]["data"].items():
588 for tst_name, tst_data in data[job][str(build)].iteritems():
589 tst_name_mod = tst_name.replace("-ndrpdr", "").\
591 if tst_name_mod in tests_lst:
593 if tst_data["type"] in ("NDRPDR", "MRR", "BMRR"):
594 if table["include-tests"] == "MRR":
595 result = tst_data["result"]["receive-rate"].avg
596 elif table["include-tests"] == "PDR":
597 result = tst_data["throughput"]["PDR"]["LOWER"]
598 elif table["include-tests"] == "NDR":
599 result = tst_data["throughput"]["NDR"]["LOWER"]
602 if result is not None:
603 tbl_dict[tst_name_mod]["ref-data"].append(
605 except (KeyError, TypeError):
609 for tst_name in tbl_dict.keys():
610 item = [tbl_dict[tst_name]["name"], ]
611 data_r = tbl_dict[tst_name]["ref-data"]
613 data_r_mean = mean(data_r)
614 item.append(round(data_r_mean / 1000000, 2))
615 data_r_stdev = stdev(data_r)
616 item.append(round(data_r_stdev / 1000000, 2))
620 item.extend([None, None])
621 data_c = tbl_dict[tst_name]["cmp-data"]
623 data_c_mean = mean(data_c)
624 item.append(round(data_c_mean / 1000000, 2))
625 data_c_stdev = stdev(data_c)
626 item.append(round(data_c_stdev / 1000000, 2))
630 item.extend([None, None])
631 if data_r_mean and data_c_mean:
632 delta, d_stdev = relative_change_stdev(
633 data_r_mean, data_c_mean, data_r_stdev, data_c_stdev)
634 item.append(round(delta, 2))
635 item.append(round(d_stdev, 2))
638 # Sort the table according to the relative change
639 tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
641 # Generate csv tables:
642 csv_file = "{0}.csv".format(table["output-file"])
643 with open(csv_file, "w") as file_handler:
644 file_handler.write(header_str)
646 file_handler.write(",".join([str(item) for item in test]) + "\n")
648 convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
651 def table_performance_trending_dashboard(table, input_data):
652 """Generate the table(s) with algorithm:
653 table_performance_trending_dashboard
654 specified in the specification file.
656 :param table: Table to generate.
657 :param input_data: Data to process.
658 :type table: pandas.Series
659 :type input_data: InputData
662 logging.info(" Generating the table {0} ...".
663 format(table.get("title", "")))
666 logging.info(" Creating the data set for the {0} '{1}'.".
667 format(table.get("type", ""), table.get("title", "")))
668 data = input_data.filter_data(table, continue_on_error=True)
670 # Prepare the header of the tables
671 header = ["Test Case",
673 "Short-Term Change [%]",
674 "Long-Term Change [%]",
678 header_str = ",".join(header) + "\n"
680 # Prepare data to the table:
682 for job, builds in table["data"].items():
684 for tst_name, tst_data in data[job][str(build)].iteritems():
685 if tst_name.lower() in table.get("ignore-list", list()):
687 if tbl_dict.get(tst_name, None) is None:
688 groups = re.search(REGEX_NIC, tst_data["parent"])
691 nic = groups.group(0)
692 tbl_dict[tst_name] = {
693 "name": "{0}-{1}".format(nic, tst_data["name"]),
694 "data": OrderedDict()}
696 tbl_dict[tst_name]["data"][str(build)] = \
697 tst_data["result"]["receive-rate"]
698 except (TypeError, KeyError):
699 pass # No data in output.xml for this test
702 for tst_name in tbl_dict.keys():
703 data_t = tbl_dict[tst_name]["data"]
707 classification_lst, avgs = classify_anomalies(data_t)
709 win_size = min(len(data_t), table["window"])
710 long_win_size = min(len(data_t), table["long-trend-window"])
714 [x for x in avgs[-long_win_size:-win_size]
719 avg_week_ago = avgs[max(-win_size, -len(avgs))]
721 if isnan(last_avg) or isnan(avg_week_ago) or avg_week_ago == 0.0:
722 rel_change_last = nan
724 rel_change_last = round(
725 ((last_avg - avg_week_ago) / avg_week_ago) * 100, 2)
727 if isnan(max_long_avg) or isnan(last_avg) or max_long_avg == 0.0:
728 rel_change_long = nan
730 rel_change_long = round(
731 ((last_avg - max_long_avg) / max_long_avg) * 100, 2)
733 if classification_lst:
734 if isnan(rel_change_last) and isnan(rel_change_long):
736 if (isnan(last_avg) or
737 isnan(rel_change_last) or
738 isnan(rel_change_long)):
741 [tbl_dict[tst_name]["name"],
742 round(last_avg / 1000000, 2),
745 classification_lst[-win_size:].count("regression"),
746 classification_lst[-win_size:].count("progression")])
748 tbl_lst.sort(key=lambda rel: rel[0])
751 for nrr in range(table["window"], -1, -1):
752 tbl_reg = [item for item in tbl_lst if item[4] == nrr]
753 for nrp in range(table["window"], -1, -1):
754 tbl_out = [item for item in tbl_reg if item[5] == nrp]
755 tbl_out.sort(key=lambda rel: rel[2])
756 tbl_sorted.extend(tbl_out)
758 file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
760 logging.info(" Writing file: '{0}'".format(file_name))
761 with open(file_name, "w") as file_handler:
762 file_handler.write(header_str)
763 for test in tbl_sorted:
764 file_handler.write(",".join([str(item) for item in test]) + '\n')
766 txt_file_name = "{0}.txt".format(table["output-file"])
767 logging.info(" Writing file: '{0}'".format(txt_file_name))
768 convert_csv_to_pretty_txt(file_name, txt_file_name)
771 def _generate_url(base, testbed, test_name):
772 """Generate URL to a trending plot from the name of the test case.
774 :param base: The base part of URL common to all test cases.
775 :param testbed: The testbed used for testing.
776 :param test_name: The name of the test case.
780 :returns: The URL to the plot with the trending data for the given test
790 if "lbdpdk" in test_name or "lbvpp" in test_name:
791 file_name = "link_bonding"
793 elif "114b" in test_name and "vhost" in test_name:
796 elif "testpmd" in test_name or "l3fwd" in test_name:
799 elif "memif" in test_name:
800 file_name = "container_memif"
803 elif "srv6" in test_name:
806 elif "vhost" in test_name:
807 if "l2xcbase" in test_name or "l2bdbasemaclrn" in test_name:
808 file_name = "vm_vhost_l2"
809 if "114b" in test_name:
811 elif "l2xcbase" in test_name and "x520" in test_name:
812 feature = "-base-l2xc"
813 elif "l2bdbasemaclrn" in test_name and "x520" in test_name:
814 feature = "-base-l2bd"
817 elif "ip4base" in test_name:
818 file_name = "vm_vhost_ip4"
821 elif "ipsecbasetnlsw" in test_name:
822 file_name = "ipsecsw"
823 feature = "-base-scale"
825 elif "ipsec" in test_name:
827 feature = "-base-scale"
828 if "-int-" in test_name:
829 feature = "-base-scale-int"
830 elif "tnl" in test_name:
831 feature = "-base-scale-tnl"
833 elif "ethip4lispip" in test_name or "ethip4vxlan" in test_name:
834 file_name = "ip4_tunnels"
837 elif "ip4base" in test_name or "ip4scale" in test_name:
839 if "xl710" in test_name:
840 feature = "-base-scale-features"
841 elif "iacl" in test_name:
842 feature = "-features-iacl"
843 elif "oacl" in test_name:
844 feature = "-features-oacl"
845 elif "snat" in test_name or "cop" in test_name:
846 feature = "-features"
848 feature = "-base-scale"
850 elif "ip6base" in test_name or "ip6scale" in test_name:
852 feature = "-base-scale"
854 elif "l2xcbase" in test_name or "l2xcscale" in test_name \
855 or "l2bdbasemaclrn" in test_name or "l2bdscale" in test_name \
856 or "l2dbbasemaclrn" in test_name or "l2dbscale" in test_name:
858 if "macip" in test_name:
859 feature = "-features-macip"
860 elif "iacl" in test_name:
861 feature = "-features-iacl"
862 elif "oacl" in test_name:
863 feature = "-features-oacl"
865 feature = "-base-scale"
867 if "x520" in test_name:
869 elif "x710" in test_name:
871 elif "xl710" in test_name:
873 elif "xxv710" in test_name:
875 elif "vic1227" in test_name:
877 elif "vic1385" in test_name:
879 elif "x553" in test_name:
885 if "64b" in test_name:
887 elif "78b" in test_name:
889 elif "imix" in test_name:
891 elif "9000b" in test_name:
893 elif "1518b" in test_name:
895 elif "114b" in test_name:
899 anchor += framesize + '-'
901 if "1t1c" in test_name:
903 elif "2t2c" in test_name:
905 elif "4t4c" in test_name:
907 elif "2t1c" in test_name:
909 elif "4t2c" in test_name:
911 elif "8t4c" in test_name:
914 return url + file_name + '-' + testbed + '-' + nic + framesize + \
915 feature.replace("-int", "").replace("-tnl", "") + anchor + feature
918 def table_performance_trending_dashboard_html(table, input_data):
919 """Generate the table(s) with algorithm:
920 table_performance_trending_dashboard_html specified in the specification
923 :param table: Table to generate.
924 :param input_data: Data to process.
926 :type input_data: InputData
929 testbed = table.get("testbed", None)
931 logging.error("The testbed is not defined for the table '{0}'.".
932 format(table.get("title", "")))
935 logging.info(" Generating the table {0} ...".
936 format(table.get("title", "")))
939 with open(table["input-file"], 'rb') as csv_file:
940 csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
941 csv_lst = [item for item in csv_content]
943 logging.warning("The input file is not defined.")
945 except csv.Error as err:
946 logging.warning("Not possible to process the file '{0}'.\n{1}".
947 format(table["input-file"], err))
951 dashboard = ET.Element("table", attrib=dict(width="100%", border='0'))
954 tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor="#7eade7"))
955 for idx, item in enumerate(csv_lst[0]):
956 alignment = "left" if idx == 0 else "center"
957 th = ET.SubElement(tr, "th", attrib=dict(align=alignment))
961 colors = {"regression": ("#ffcccc", "#ff9999"),
962 "progression": ("#c6ecc6", "#9fdf9f"),
963 "normal": ("#e9f1fb", "#d4e4f7")}
964 for r_idx, row in enumerate(csv_lst[1:]):
968 color = "progression"
971 background = colors[color][r_idx % 2]
972 tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor=background))
975 for c_idx, item in enumerate(row):
976 alignment = "left" if c_idx == 0 else "center"
977 td = ET.SubElement(tr, "td", attrib=dict(align=alignment))
980 url = _generate_url("../trending/", testbed, item)
981 ref = ET.SubElement(td, "a", attrib=dict(href=url))
986 with open(table["output-file"], 'w') as html_file:
987 logging.info(" Writing file: '{0}'".format(table["output-file"]))
988 html_file.write(".. raw:: html\n\n\t")
989 html_file.write(ET.tostring(dashboard))
990 html_file.write("\n\t<p><br><br></p>\n")
992 logging.warning("The output file is not defined.")
996 def table_last_failed_tests(table, input_data):
997 """Generate the table(s) with algorithm: table_last_failed_tests
998 specified in the specification file.
1000 :param table: Table to generate.
1001 :param input_data: Data to process.
1002 :type table: pandas.Series
1003 :type input_data: InputData
1006 logging.info(" Generating the table {0} ...".
1007 format(table.get("title", "")))
1009 # Transform the data
1010 logging.info(" Creating the data set for the {0} '{1}'.".
1011 format(table.get("type", ""), table.get("title", "")))
1012 data = input_data.filter_data(table, continue_on_error=True)
1014 if data is None or data.empty:
1015 logging.warn(" No data for the {0} '{1}'.".
1016 format(table.get("type", ""), table.get("title", "")))
1020 for job, builds in table["data"].items():
1021 for build in builds:
1024 version = input_data.metadata(job, build).get("version", "")
1026 logging.error("Data for {job}: {build} is not present.".
1027 format(job=job, build=build))
1029 tbl_list.append(build)
1030 tbl_list.append(version)
1031 for tst_name, tst_data in data[job][build].iteritems():
1032 if tst_data["status"] != "FAIL":
1034 groups = re.search(REGEX_NIC, tst_data["parent"])
1037 nic = groups.group(0)
1038 tbl_list.append("{0}-{1}".format(nic, tst_data["name"]))
1040 file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
1041 logging.info(" Writing file: '{0}'".format(file_name))
1042 with open(file_name, "w") as file_handler:
1043 for test in tbl_list:
1044 file_handler.write(test + '\n')
1047 def table_failed_tests(table, input_data):
1048 """Generate the table(s) with algorithm: table_failed_tests
1049 specified in the specification file.
1051 :param table: Table to generate.
1052 :param input_data: Data to process.
1053 :type table: pandas.Series
1054 :type input_data: InputData
1057 logging.info(" Generating the table {0} ...".
1058 format(table.get("title", "")))
1060 # Transform the data
1061 logging.info(" Creating the data set for the {0} '{1}'.".
1062 format(table.get("type", ""), table.get("title", "")))
1063 data = input_data.filter_data(table, continue_on_error=True)
1065 # Prepare the header of the tables
1066 header = ["Test Case",
1068 "Last Failure [Time]",
1069 "Last Failure [VPP-Build-Id]",
1070 "Last Failure [CSIT-Job-Build-Id]"]
1072 # Generate the data for the table according to the model in the table
1076 timeperiod = timedelta(int(table.get("window", 7)))
1079 for job, builds in table["data"].items():
1080 for build in builds:
1082 for tst_name, tst_data in data[job][build].iteritems():
1083 if tst_name.lower() in table.get("ignore-list", list()):
1085 if tbl_dict.get(tst_name, None) is None:
1086 groups = re.search(REGEX_NIC, tst_data["parent"])
1089 nic = groups.group(0)
1090 tbl_dict[tst_name] = {
1091 "name": "{0}-{1}".format(nic, tst_data["name"]),
1092 "data": OrderedDict()}
1094 generated = input_data.metadata(job, build).\
1095 get("generated", "")
1098 then = dt.strptime(generated, "%Y%m%d %H:%M")
1099 if (now - then) <= timeperiod:
1100 tbl_dict[tst_name]["data"][build] = (
1103 input_data.metadata(job, build).get("version", ""),
1105 except (TypeError, KeyError) as err:
1106 logging.warning("tst_name: {} - err: {}".
1107 format(tst_name, repr(err)))
1111 for tst_data in tbl_dict.values():
1113 for val in tst_data["data"].values():
1114 if val[0] == "FAIL":
1116 fails_last_date = val[1]
1117 fails_last_vpp = val[2]
1118 fails_last_csit = val[3]
1120 max_fails = fails_nr if fails_nr > max_fails else max_fails
1121 tbl_lst.append([tst_data["name"],
1125 "mrr-daily-build-{0}".format(fails_last_csit)])
1127 tbl_lst.sort(key=lambda rel: rel[2], reverse=True)
1129 for nrf in range(max_fails, -1, -1):
1130 tbl_fails = [item for item in tbl_lst if item[1] == nrf]
1131 tbl_sorted.extend(tbl_fails)
1132 file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
1134 logging.info(" Writing file: '{0}'".format(file_name))
1135 with open(file_name, "w") as file_handler:
1136 file_handler.write(",".join(header) + "\n")
1137 for test in tbl_sorted:
1138 file_handler.write(",".join([str(item) for item in test]) + '\n')
1140 txt_file_name = "{0}.txt".format(table["output-file"])
1141 logging.info(" Writing file: '{0}'".format(txt_file_name))
1142 convert_csv_to_pretty_txt(file_name, txt_file_name)
1145 def table_failed_tests_html(table, input_data):
1146 """Generate the table(s) with algorithm: table_failed_tests_html
1147 specified in the specification file.
1149 :param table: Table to generate.
1150 :param input_data: Data to process.
1151 :type table: pandas.Series
1152 :type input_data: InputData
1155 testbed = table.get("testbed", None)
1157 logging.error("The testbed is not defined for the table '{0}'.".
1158 format(table.get("title", "")))
1161 logging.info(" Generating the table {0} ...".
1162 format(table.get("title", "")))
1165 with open(table["input-file"], 'rb') as csv_file:
1166 csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
1167 csv_lst = [item for item in csv_content]
1169 logging.warning("The input file is not defined.")
1171 except csv.Error as err:
1172 logging.warning("Not possible to process the file '{0}'.\n{1}".
1173 format(table["input-file"], err))
1177 failed_tests = ET.Element("table", attrib=dict(width="100%", border='0'))
1180 tr = ET.SubElement(failed_tests, "tr", attrib=dict(bgcolor="#7eade7"))
1181 for idx, item in enumerate(csv_lst[0]):
1182 alignment = "left" if idx == 0 else "center"
1183 th = ET.SubElement(tr, "th", attrib=dict(align=alignment))
1187 colors = ("#e9f1fb", "#d4e4f7")
1188 for r_idx, row in enumerate(csv_lst[1:]):
1189 background = colors[r_idx % 2]
1190 tr = ET.SubElement(failed_tests, "tr", attrib=dict(bgcolor=background))
1193 for c_idx, item in enumerate(row):
1194 alignment = "left" if c_idx == 0 else "center"
1195 td = ET.SubElement(tr, "td", attrib=dict(align=alignment))
1198 url = _generate_url("../trending/", testbed, item)
1199 ref = ET.SubElement(td, "a", attrib=dict(href=url))
1204 with open(table["output-file"], 'w') as html_file:
1205 logging.info(" Writing file: '{0}'".format(table["output-file"]))
1206 html_file.write(".. raw:: html\n\n\t")
1207 html_file.write(ET.tostring(failed_tests))
1208 html_file.write("\n\t<p><br><br></p>\n")
1210 logging.warning("The output file is not defined.")