1 # Copyright (c) 2019 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Algorithms to generate tables.
22 from string import replace
23 from collections import OrderedDict
24 from numpy import nan, isnan
25 from xml.etree import ElementTree as ET
26 from datetime import datetime as dt
27 from datetime import timedelta
29 from utils import mean, stdev, relative_change, classify_anomalies, \
30 convert_csv_to_pretty_txt
33 REGEX_NIC = re.compile(r'\d*ge\dp\d\D*\d*')
36 def generate_tables(spec, data):
37 """Generate all tables specified in the specification file.
39 :param spec: Specification read from the specification file.
40 :param data: Data to process.
41 :type spec: Specification
45 logging.info("Generating the tables ...")
46 for table in spec.tables:
48 eval(table["algorithm"])(table, data)
49 except NameError as err:
50 logging.error("Probably algorithm '{alg}' is not defined: {err}".
51 format(alg=table["algorithm"], err=repr(err)))
55 def table_details(table, input_data):
56 """Generate the table(s) with algorithm: table_detailed_test_results
57 specified in the specification file.
59 :param table: Table to generate.
60 :param input_data: Data to process.
61 :type table: pandas.Series
62 :type input_data: InputData
65 logging.info(" Generating the table {0} ...".
66 format(table.get("title", "")))
69 logging.info(" Creating the data set for the {0} '{1}'.".
70 format(table.get("type", ""), table.get("title", "")))
71 data = input_data.filter_data(table)
73 # Prepare the header of the tables
75 for column in table["columns"]:
76 header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
78 # Generate the data for the table according to the model in the table
80 job = table["data"].keys()[0]
81 build = str(table["data"][job][0])
83 suites = input_data.suites(job, build)
85 logging.error(" No data available. The table will not be generated.")
88 for suite_longname, suite in suites.iteritems():
90 suite_name = suite["name"]
92 for test in data[job][build].keys():
93 if data[job][build][test]["parent"] in suite_name:
95 for column in table["columns"]:
97 col_data = str(data[job][build][test][column["data"].
98 split(" ")[1]]).replace('"', '""')
99 if column["data"].split(" ")[1] in ("conf-history",
101 col_data = replace(col_data, " |br| ", "",
103 col_data = " |prein| {0} |preout| ".\
104 format(col_data[:-5])
105 row_lst.append('"{0}"'.format(col_data))
107 row_lst.append("No data")
108 table_lst.append(row_lst)
110 # Write the data to file
112 file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
113 table["output-file-ext"])
114 logging.info(" Writing file: '{}'".format(file_name))
115 with open(file_name, "w") as file_handler:
116 file_handler.write(",".join(header) + "\n")
117 for item in table_lst:
118 file_handler.write(",".join(item) + "\n")
120 logging.info(" Done.")
123 def table_merged_details(table, input_data):
124 """Generate the table(s) with algorithm: table_merged_details
125 specified in the specification file.
127 :param table: Table to generate.
128 :param input_data: Data to process.
129 :type table: pandas.Series
130 :type input_data: InputData
133 logging.info(" Generating the table {0} ...".
134 format(table.get("title", "")))
137 logging.info(" Creating the data set for the {0} '{1}'.".
138 format(table.get("type", ""), table.get("title", "")))
139 data = input_data.filter_data(table)
140 data = input_data.merge_data(data)
141 data.sort_index(inplace=True)
143 logging.info(" Creating the data set for the {0} '{1}'.".
144 format(table.get("type", ""), table.get("title", "")))
145 suites = input_data.filter_data(table, data_set="suites")
146 suites = input_data.merge_data(suites)
148 # Prepare the header of the tables
150 for column in table["columns"]:
151 header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
153 for _, suite in suites.iteritems():
155 suite_name = suite["name"]
157 for test in data.keys():
158 if data[test]["parent"] in suite_name:
160 for column in table["columns"]:
162 col_data = str(data[test][column["data"].
163 split(" ")[1]]).replace('"', '""')
164 col_data = replace(col_data, "No Data",
166 if column["data"].split(" ")[1] in ("conf-history",
168 col_data = replace(col_data, " |br| ", "",
170 col_data = " |prein| {0} |preout| ".\
171 format(col_data[:-5])
172 row_lst.append('"{0}"'.format(col_data))
174 row_lst.append('"Not captured"')
175 table_lst.append(row_lst)
177 # Write the data to file
179 file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
180 table["output-file-ext"])
181 logging.info(" Writing file: '{}'".format(file_name))
182 with open(file_name, "w") as file_handler:
183 file_handler.write(",".join(header) + "\n")
184 for item in table_lst:
185 file_handler.write(",".join(item) + "\n")
187 logging.info(" Done.")
190 def table_performance_comparison(table, input_data):
191 """Generate the table(s) with algorithm: table_performance_comparison
192 specified in the specification file.
194 :param table: Table to generate.
195 :param input_data: Data to process.
196 :type table: pandas.Series
197 :type input_data: InputData
200 logging.info(" Generating the table {0} ...".
201 format(table.get("title", "")))
204 logging.info(" Creating the data set for the {0} '{1}'.".
205 format(table.get("type", ""), table.get("title", "")))
206 data = input_data.filter_data(table, continue_on_error=True)
208 # Prepare the header of the tables
210 header = ["Test case", ]
212 if table["include-tests"] == "MRR":
213 hdr_param = "Receive Rate"
215 hdr_param = "Throughput"
217 history = table.get("history", None)
221 ["{0} {1} [Mpps]".format(item["title"], hdr_param),
222 "{0} Stdev [Mpps]".format(item["title"])])
224 ["{0} {1} [Mpps]".format(table["reference"]["title"], hdr_param),
225 "{0} Stdev [Mpps]".format(table["reference"]["title"]),
226 "{0} {1} [Mpps]".format(table["compare"]["title"], hdr_param),
227 "{0} Stdev [Mpps]".format(table["compare"]["title"]),
229 header_str = ",".join(header) + "\n"
230 except (AttributeError, KeyError) as err:
231 logging.error("The model is invalid, missing parameter: {0}".
235 # Prepare data to the table:
237 for job, builds in table["reference"]["data"].items():
239 for tst_name, tst_data in data[job][str(build)].iteritems():
240 tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\
241 replace("-ndrpdr", "").replace("-pdrdisc", "").\
242 replace("-ndrdisc", "").replace("-pdr", "").\
243 replace("-ndr", "").\
244 replace("1t1c", "1c").replace("2t1c", "1c").\
245 replace("2t2c", "2c").replace("4t2c", "2c").\
246 replace("4t4c", "4c").replace("8t4c", "4c")
247 if "across topologies" in table["title"].lower():
248 tst_name_mod = tst_name_mod.replace("2n1l-", "")
249 if tbl_dict.get(tst_name_mod, None) is None:
250 groups = re.search(REGEX_NIC, tst_data["parent"])
251 nic = groups.group(0) if groups else ""
252 name = "{0}-{1}".format(nic, "-".join(tst_data["name"].
254 if "across testbeds" in table["title"].lower() or \
255 "across topologies" in table["title"].lower():
257 replace("1t1c", "1c").replace("2t1c", "1c").\
258 replace("2t2c", "2c").replace("4t2c", "2c").\
259 replace("4t4c", "4c").replace("8t4c", "4c")
260 tbl_dict[tst_name_mod] = {"name": name,
264 # TODO: Re-work when NDRPDRDISC tests are not used
265 if table["include-tests"] == "MRR":
266 tbl_dict[tst_name_mod]["ref-data"]. \
267 append(tst_data["result"]["receive-rate"].avg)
268 elif table["include-tests"] == "PDR":
269 if tst_data["type"] == "PDR":
270 tbl_dict[tst_name_mod]["ref-data"]. \
271 append(tst_data["throughput"]["value"])
272 elif tst_data["type"] == "NDRPDR":
273 tbl_dict[tst_name_mod]["ref-data"].append(
274 tst_data["throughput"]["PDR"]["LOWER"])
275 elif table["include-tests"] == "NDR":
276 if tst_data["type"] == "NDR":
277 tbl_dict[tst_name_mod]["ref-data"]. \
278 append(tst_data["throughput"]["value"])
279 elif tst_data["type"] == "NDRPDR":
280 tbl_dict[tst_name_mod]["ref-data"].append(
281 tst_data["throughput"]["NDR"]["LOWER"])
285 pass # No data in output.xml for this test
287 for job, builds in table["compare"]["data"].items():
289 for tst_name, tst_data in data[job][str(build)].iteritems():
290 tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \
291 replace("-ndrpdr", "").replace("-pdrdisc", ""). \
292 replace("-ndrdisc", "").replace("-pdr", ""). \
293 replace("-ndr", "").\
294 replace("1t1c", "1c").replace("2t1c", "1c").\
295 replace("2t2c", "2c").replace("4t2c", "2c").\
296 replace("4t4c", "4c").replace("8t4c", "4c")
297 if "across topologies" in table["title"].lower():
298 tst_name_mod = tst_name_mod.replace("2n1l-", "")
300 # TODO: Re-work when NDRPDRDISC tests are not used
301 if table["include-tests"] == "MRR":
302 tbl_dict[tst_name_mod]["cmp-data"]. \
303 append(tst_data["result"]["receive-rate"].avg)
304 elif table["include-tests"] == "PDR":
305 if tst_data["type"] == "PDR":
306 tbl_dict[tst_name_mod]["cmp-data"]. \
307 append(tst_data["throughput"]["value"])
308 elif tst_data["type"] == "NDRPDR":
309 tbl_dict[tst_name_mod]["cmp-data"].append(
310 tst_data["throughput"]["PDR"]["LOWER"])
311 elif table["include-tests"] == "NDR":
312 if tst_data["type"] == "NDR":
313 tbl_dict[tst_name_mod]["cmp-data"]. \
314 append(tst_data["throughput"]["value"])
315 elif tst_data["type"] == "NDRPDR":
316 tbl_dict[tst_name_mod]["cmp-data"].append(
317 tst_data["throughput"]["NDR"]["LOWER"])
323 tbl_dict.pop(tst_name_mod, None)
326 for job, builds in item["data"].items():
328 for tst_name, tst_data in data[job][str(build)].iteritems():
329 tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \
330 replace("-ndrpdr", "").replace("-pdrdisc", ""). \
331 replace("-ndrdisc", "").replace("-pdr", ""). \
332 replace("-ndr", "").\
333 replace("1t1c", "1c").replace("2t1c", "1c").\
334 replace("2t2c", "2c").replace("4t2c", "2c").\
335 replace("4t4c", "4c").replace("8t4c", "4c")
336 if "across topologies" in table["title"].lower():
337 tst_name_mod = tst_name_mod.replace("2n1l-", "")
338 if tbl_dict.get(tst_name_mod, None) is None:
340 if tbl_dict[tst_name_mod].get("history", None) is None:
341 tbl_dict[tst_name_mod]["history"] = OrderedDict()
342 if tbl_dict[tst_name_mod]["history"].get(item["title"],
344 tbl_dict[tst_name_mod]["history"][item["title"]] = \
347 # TODO: Re-work when NDRPDRDISC tests are not used
348 if table["include-tests"] == "MRR":
349 tbl_dict[tst_name_mod]["history"][item["title"
350 ]].append(tst_data["result"]["receive-rate"].
352 elif table["include-tests"] == "PDR":
353 if tst_data["type"] == "PDR":
354 tbl_dict[tst_name_mod]["history"][
356 append(tst_data["throughput"]["value"])
357 elif tst_data["type"] == "NDRPDR":
358 tbl_dict[tst_name_mod]["history"][item[
359 "title"]].append(tst_data["throughput"][
361 elif table["include-tests"] == "NDR":
362 if tst_data["type"] == "NDR":
363 tbl_dict[tst_name_mod]["history"][
365 append(tst_data["throughput"]["value"])
366 elif tst_data["type"] == "NDRPDR":
367 tbl_dict[tst_name_mod]["history"][item[
368 "title"]].append(tst_data["throughput"][
372 except (TypeError, KeyError):
376 for tst_name in tbl_dict.keys():
377 item = [tbl_dict[tst_name]["name"], ]
379 if tbl_dict[tst_name].get("history", None) is not None:
380 for hist_data in tbl_dict[tst_name]["history"].values():
382 item.append(round(mean(hist_data) / 1000000, 2))
383 item.append(round(stdev(hist_data) / 1000000, 2))
385 item.extend([None, None])
387 item.extend([None, None])
388 data_t = tbl_dict[tst_name]["ref-data"]
390 item.append(round(mean(data_t) / 1000000, 2))
391 item.append(round(stdev(data_t) / 1000000, 2))
393 item.extend([None, None])
394 data_t = tbl_dict[tst_name]["cmp-data"]
396 item.append(round(mean(data_t) / 1000000, 2))
397 item.append(round(stdev(data_t) / 1000000, 2))
399 item.extend([None, None])
400 if item[-4] is not None and item[-2] is not None and item[-4] != 0:
401 item.append(int(relative_change(float(item[-4]), float(item[-2]))))
402 if len(item) == len(header):
405 # Sort the table according to the relative change
406 tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
408 # Generate csv tables:
409 csv_file = "{0}.csv".format(table["output-file"])
410 with open(csv_file, "w") as file_handler:
411 file_handler.write(header_str)
413 file_handler.write(",".join([str(item) for item in test]) + "\n")
415 convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
418 def table_nics_comparison(table, input_data):
419 """Generate the table(s) with algorithm: table_nics_comparison
420 specified in the specification file.
422 :param table: Table to generate.
423 :param input_data: Data to process.
424 :type table: pandas.Series
425 :type input_data: InputData
428 logging.info(" Generating the table {0} ...".
429 format(table.get("title", "")))
432 logging.info(" Creating the data set for the {0} '{1}'.".
433 format(table.get("type", ""), table.get("title", "")))
434 data = input_data.filter_data(table, continue_on_error=True)
436 # Prepare the header of the tables
438 header = ["Test case", ]
440 if table["include-tests"] == "MRR":
441 hdr_param = "Receive Rate"
443 hdr_param = "Throughput"
446 ["{0} {1} [Mpps]".format(table["reference"]["title"], hdr_param),
447 "{0} Stdev [Mpps]".format(table["reference"]["title"]),
448 "{0} {1} [Mpps]".format(table["compare"]["title"], hdr_param),
449 "{0} Stdev [Mpps]".format(table["compare"]["title"]),
451 header_str = ",".join(header) + "\n"
452 except (AttributeError, KeyError) as err:
453 logging.error("The model is invalid, missing parameter: {0}".
457 # Prepare data to the table:
459 for job, builds in table["data"].items():
461 for tst_name, tst_data in data[job][str(build)].iteritems():
462 tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\
463 replace("-ndrpdr", "").replace("-pdrdisc", "").\
464 replace("-ndrdisc", "").replace("-pdr", "").\
465 replace("-ndr", "").\
466 replace("1t1c", "1c").replace("2t1c", "1c").\
467 replace("2t2c", "2c").replace("4t2c", "2c").\
468 replace("4t4c", "4c").replace("8t4c", "4c")
469 tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod)
470 if tbl_dict.get(tst_name_mod, None) is None:
471 name = "-".join(tst_data["name"].split("-")[:-1])
472 tbl_dict[tst_name_mod] = {"name": name,
476 if table["include-tests"] == "MRR":
477 result = tst_data["result"]["receive-rate"].avg
478 elif table["include-tests"] == "PDR":
479 result = tst_data["throughput"]["PDR"]["LOWER"]
480 elif table["include-tests"] == "NDR":
481 result = tst_data["throughput"]["NDR"]["LOWER"]
486 if table["reference"]["nic"] in tst_data["tags"]:
487 tbl_dict[tst_name_mod]["ref-data"].append(result)
488 elif table["compare"]["nic"] in tst_data["tags"]:
489 tbl_dict[tst_name_mod]["cmp-data"].append(result)
490 except (TypeError, KeyError) as err:
491 logging.debug("No data for {0}".format(tst_name))
492 logging.debug(repr(err))
493 # No data in output.xml for this test
496 for tst_name in tbl_dict.keys():
497 item = [tbl_dict[tst_name]["name"], ]
498 data_t = tbl_dict[tst_name]["ref-data"]
500 item.append(round(mean(data_t) / 1000000, 2))
501 item.append(round(stdev(data_t) / 1000000, 2))
503 item.extend([None, None])
504 data_t = tbl_dict[tst_name]["cmp-data"]
506 item.append(round(mean(data_t) / 1000000, 2))
507 item.append(round(stdev(data_t) / 1000000, 2))
509 item.extend([None, None])
510 if item[-4] is not None and item[-2] is not None and item[-4] != 0:
511 item.append(int(relative_change(float(item[-4]), float(item[-2]))))
512 if len(item) == len(header):
515 # Sort the table according to the relative change
516 tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
518 # Generate csv tables:
519 csv_file = "{0}.csv".format(table["output-file"])
520 with open(csv_file, "w") as file_handler:
521 file_handler.write(header_str)
523 file_handler.write(",".join([str(item) for item in test]) + "\n")
525 convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
528 def table_performance_trending_dashboard(table, input_data):
529 """Generate the table(s) with algorithm:
530 table_performance_trending_dashboard
531 specified in the specification file.
533 :param table: Table to generate.
534 :param input_data: Data to process.
535 :type table: pandas.Series
536 :type input_data: InputData
539 logging.info(" Generating the table {0} ...".
540 format(table.get("title", "")))
543 logging.info(" Creating the data set for the {0} '{1}'.".
544 format(table.get("type", ""), table.get("title", "")))
545 data = input_data.filter_data(table, continue_on_error=True)
547 # Prepare the header of the tables
548 header = ["Test Case",
550 "Short-Term Change [%]",
551 "Long-Term Change [%]",
555 header_str = ",".join(header) + "\n"
557 # Prepare data to the table:
559 for job, builds in table["data"].items():
561 for tst_name, tst_data in data[job][str(build)].iteritems():
562 if tst_name.lower() in table["ignore-list"]:
564 if tbl_dict.get(tst_name, None) is None:
565 groups = re.search(REGEX_NIC, tst_data["parent"])
568 nic = groups.group(0)
569 tbl_dict[tst_name] = {
570 "name": "{0}-{1}".format(nic, tst_data["name"]),
571 "data": OrderedDict()}
573 tbl_dict[tst_name]["data"][str(build)] = \
574 tst_data["result"]["receive-rate"]
575 except (TypeError, KeyError):
576 pass # No data in output.xml for this test
579 for tst_name in tbl_dict.keys():
580 data_t = tbl_dict[tst_name]["data"]
584 classification_lst, avgs = classify_anomalies(data_t)
586 win_size = min(len(data_t), table["window"])
587 long_win_size = min(len(data_t), table["long-trend-window"])
591 [x for x in avgs[-long_win_size:-win_size]
596 avg_week_ago = avgs[max(-win_size, -len(avgs))]
598 if isnan(last_avg) or isnan(avg_week_ago) or avg_week_ago == 0.0:
599 rel_change_last = nan
601 rel_change_last = round(
602 ((last_avg - avg_week_ago) / avg_week_ago) * 100, 2)
604 if isnan(max_long_avg) or isnan(last_avg) or max_long_avg == 0.0:
605 rel_change_long = nan
607 rel_change_long = round(
608 ((last_avg - max_long_avg) / max_long_avg) * 100, 2)
610 if classification_lst:
611 if isnan(rel_change_last) and isnan(rel_change_long):
613 if (isnan(last_avg) or
614 isnan(rel_change_last) or
615 isnan(rel_change_long)):
618 [tbl_dict[tst_name]["name"],
619 round(last_avg / 1000000, 2),
622 classification_lst[-win_size:].count("regression"),
623 classification_lst[-win_size:].count("progression")])
625 tbl_lst.sort(key=lambda rel: rel[0])
628 for nrr in range(table["window"], -1, -1):
629 tbl_reg = [item for item in tbl_lst if item[4] == nrr]
630 for nrp in range(table["window"], -1, -1):
631 tbl_out = [item for item in tbl_reg if item[5] == nrp]
632 tbl_out.sort(key=lambda rel: rel[2])
633 tbl_sorted.extend(tbl_out)
635 file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
637 logging.info(" Writing file: '{0}'".format(file_name))
638 with open(file_name, "w") as file_handler:
639 file_handler.write(header_str)
640 for test in tbl_sorted:
641 file_handler.write(",".join([str(item) for item in test]) + '\n')
643 txt_file_name = "{0}.txt".format(table["output-file"])
644 logging.info(" Writing file: '{0}'".format(txt_file_name))
645 convert_csv_to_pretty_txt(file_name, txt_file_name)
648 def _generate_url(base, testbed, test_name):
649 """Generate URL to a trending plot from the name of the test case.
651 :param base: The base part of URL common to all test cases.
652 :param testbed: The testbed used for testing.
653 :param test_name: The name of the test case.
657 :returns: The URL to the plot with the trending data for the given test
667 if "lbdpdk" in test_name or "lbvpp" in test_name:
668 file_name = "link_bonding"
670 elif "114b" in test_name and "vhost" in test_name:
673 elif "testpmd" in test_name or "l3fwd" in test_name:
676 elif "memif" in test_name:
677 file_name = "container_memif"
680 elif "srv6" in test_name:
683 elif "vhost" in test_name:
684 if "l2xcbase" in test_name or "l2bdbasemaclrn" in test_name:
685 file_name = "vm_vhost_l2"
686 if "114b" in test_name:
688 elif "l2xcbase" in test_name and "x520" in test_name:
689 feature = "-base-l2xc"
690 elif "l2bdbasemaclrn" in test_name and "x520" in test_name:
691 feature = "-base-l2bd"
694 elif "ip4base" in test_name:
695 file_name = "vm_vhost_ip4"
698 elif "ipsec" in test_name:
700 feature = "-base-scale"
702 elif "ethip4lispip" in test_name or "ethip4vxlan" in test_name:
703 file_name = "ip4_tunnels"
706 elif "ip4base" in test_name or "ip4scale" in test_name:
708 if "xl710" in test_name:
709 feature = "-base-scale-features"
710 elif "iacl" in test_name:
711 feature = "-features-iacl"
712 elif "oacl" in test_name:
713 feature = "-features-oacl"
714 elif "snat" in test_name or "cop" in test_name:
715 feature = "-features"
717 feature = "-base-scale"
719 elif "ip6base" in test_name or "ip6scale" in test_name:
721 feature = "-base-scale"
723 elif "l2xcbase" in test_name or "l2xcscale" in test_name \
724 or "l2bdbasemaclrn" in test_name or "l2bdscale" in test_name \
725 or "l2dbbasemaclrn" in test_name or "l2dbscale" in test_name:
727 if "macip" in test_name:
728 feature = "-features-macip"
729 elif "iacl" in test_name:
730 feature = "-features-iacl"
731 elif "oacl" in test_name:
732 feature = "-features-oacl"
734 feature = "-base-scale"
736 if "x520" in test_name:
738 elif "x710" in test_name:
740 elif "xl710" in test_name:
742 elif "xxv710" in test_name:
744 elif "vic1227" in test_name:
746 elif "vic1385" in test_name:
752 if "64b" in test_name:
754 elif "78b" in test_name:
756 elif "imix" in test_name:
758 elif "9000b" in test_name:
760 elif "1518b" in test_name:
762 elif "114b" in test_name:
766 anchor += framesize + '-'
768 if "1t1c" in test_name:
770 elif "2t2c" in test_name:
772 elif "4t4c" in test_name:
774 elif "2t1c" in test_name:
776 elif "4t2c" in test_name:
778 elif "8t4c" in test_name:
781 return url + file_name + '-' + testbed + '-' + nic + framesize + feature + \
785 def table_performance_trending_dashboard_html(table, input_data):
786 """Generate the table(s) with algorithm:
787 table_performance_trending_dashboard_html specified in the specification
790 :param table: Table to generate.
791 :param input_data: Data to process.
793 :type input_data: InputData
796 testbed = table.get("testbed", None)
798 logging.error("The testbed is not defined for the table '{0}'.".
799 format(table.get("title", "")))
802 logging.info(" Generating the table {0} ...".
803 format(table.get("title", "")))
806 with open(table["input-file"], 'rb') as csv_file:
807 csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
808 csv_lst = [item for item in csv_content]
810 logging.warning("The input file is not defined.")
812 except csv.Error as err:
813 logging.warning("Not possible to process the file '{0}'.\n{1}".
814 format(table["input-file"], err))
818 dashboard = ET.Element("table", attrib=dict(width="100%", border='0'))
821 tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor="#7eade7"))
822 for idx, item in enumerate(csv_lst[0]):
823 alignment = "left" if idx == 0 else "center"
824 th = ET.SubElement(tr, "th", attrib=dict(align=alignment))
828 colors = {"regression": ("#ffcccc", "#ff9999"),
829 "progression": ("#c6ecc6", "#9fdf9f"),
830 "normal": ("#e9f1fb", "#d4e4f7")}
831 for r_idx, row in enumerate(csv_lst[1:]):
835 color = "progression"
838 background = colors[color][r_idx % 2]
839 tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor=background))
842 for c_idx, item in enumerate(row):
843 alignment = "left" if c_idx == 0 else "center"
844 td = ET.SubElement(tr, "td", attrib=dict(align=alignment))
847 url = _generate_url("../trending/", testbed, item)
848 ref = ET.SubElement(td, "a", attrib=dict(href=url))
853 with open(table["output-file"], 'w') as html_file:
854 logging.info(" Writing file: '{0}'".format(table["output-file"]))
855 html_file.write(".. raw:: html\n\n\t")
856 html_file.write(ET.tostring(dashboard))
857 html_file.write("\n\t<p><br><br></p>\n")
859 logging.warning("The output file is not defined.")
863 def table_last_failed_tests(table, input_data):
864 """Generate the table(s) with algorithm: table_last_failed_tests
865 specified in the specification file.
867 :param table: Table to generate.
868 :param input_data: Data to process.
869 :type table: pandas.Series
870 :type input_data: InputData
873 logging.info(" Generating the table {0} ...".
874 format(table.get("title", "")))
877 logging.info(" Creating the data set for the {0} '{1}'.".
878 format(table.get("type", ""), table.get("title", "")))
879 data = input_data.filter_data(table, continue_on_error=True)
881 if data is None or data.empty:
882 logging.warn(" No data for the {0} '{1}'.".
883 format(table.get("type", ""), table.get("title", "")))
887 for job, builds in table["data"].items():
891 version = input_data.metadata(job, build).get("version", "")
893 logging.error("Data for {job}: {build} is not present.".
894 format(job=job, build=build))
896 tbl_list.append(build)
897 tbl_list.append(version)
898 for tst_name, tst_data in data[job][build].iteritems():
899 if tst_data["status"] != "FAIL":
901 groups = re.search(REGEX_NIC, tst_data["parent"])
904 nic = groups.group(0)
905 tbl_list.append("{0}-{1}".format(nic, tst_data["name"]))
907 file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
908 logging.info(" Writing file: '{0}'".format(file_name))
909 with open(file_name, "w") as file_handler:
910 for test in tbl_list:
911 file_handler.write(test + '\n')
914 def table_failed_tests(table, input_data):
915 """Generate the table(s) with algorithm: table_failed_tests
916 specified in the specification file.
918 :param table: Table to generate.
919 :param input_data: Data to process.
920 :type table: pandas.Series
921 :type input_data: InputData
924 logging.info(" Generating the table {0} ...".
925 format(table.get("title", "")))
928 logging.info(" Creating the data set for the {0} '{1}'.".
929 format(table.get("type", ""), table.get("title", "")))
930 data = input_data.filter_data(table, continue_on_error=True)
932 # Prepare the header of the tables
933 header = ["Test Case",
935 "Last Failure [Time]",
936 "Last Failure [VPP-Build-Id]",
937 "Last Failure [CSIT-Job-Build-Id]"]
939 # Generate the data for the table according to the model in the table
943 timeperiod = timedelta(int(table.get("window", 7)))
946 for job, builds in table["data"].items():
949 for tst_name, tst_data in data[job][build].iteritems():
950 if tst_name.lower() in table["ignore-list"]:
952 if tbl_dict.get(tst_name, None) is None:
953 groups = re.search(REGEX_NIC, tst_data["parent"])
956 nic = groups.group(0)
957 tbl_dict[tst_name] = {
958 "name": "{0}-{1}".format(nic, tst_data["name"]),
959 "data": OrderedDict()}
961 generated = input_data.metadata(job, build).\
965 then = dt.strptime(generated, "%Y%m%d %H:%M")
966 if (now - then) <= timeperiod:
967 tbl_dict[tst_name]["data"][build] = (
970 input_data.metadata(job, build).get("version", ""),
972 except (TypeError, KeyError) as err:
973 logging.warning("tst_name: {} - err: {}".
974 format(tst_name, repr(err)))
978 for tst_data in tbl_dict.values():
980 for val in tst_data["data"].values():
983 fails_last_date = val[1]
984 fails_last_vpp = val[2]
985 fails_last_csit = val[3]
987 max_fails = fails_nr if fails_nr > max_fails else max_fails
988 tbl_lst.append([tst_data["name"],
992 "mrr-daily-build-{0}".format(fails_last_csit)])
994 tbl_lst.sort(key=lambda rel: rel[2], reverse=True)
996 for nrf in range(max_fails, -1, -1):
997 tbl_fails = [item for item in tbl_lst if item[1] == nrf]
998 tbl_sorted.extend(tbl_fails)
999 file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
1001 logging.info(" Writing file: '{0}'".format(file_name))
1002 with open(file_name, "w") as file_handler:
1003 file_handler.write(",".join(header) + "\n")
1004 for test in tbl_sorted:
1005 file_handler.write(",".join([str(item) for item in test]) + '\n')
1007 txt_file_name = "{0}.txt".format(table["output-file"])
1008 logging.info(" Writing file: '{0}'".format(txt_file_name))
1009 convert_csv_to_pretty_txt(file_name, txt_file_name)
1012 def table_failed_tests_html(table, input_data):
1013 """Generate the table(s) with algorithm: table_failed_tests_html
1014 specified in the specification file.
1016 :param table: Table to generate.
1017 :param input_data: Data to process.
1018 :type table: pandas.Series
1019 :type input_data: InputData
1022 testbed = table.get("testbed", None)
1024 logging.error("The testbed is not defined for the table '{0}'.".
1025 format(table.get("title", "")))
1028 logging.info(" Generating the table {0} ...".
1029 format(table.get("title", "")))
1032 with open(table["input-file"], 'rb') as csv_file:
1033 csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
1034 csv_lst = [item for item in csv_content]
1036 logging.warning("The input file is not defined.")
1038 except csv.Error as err:
1039 logging.warning("Not possible to process the file '{0}'.\n{1}".
1040 format(table["input-file"], err))
1044 failed_tests = ET.Element("table", attrib=dict(width="100%", border='0'))
1047 tr = ET.SubElement(failed_tests, "tr", attrib=dict(bgcolor="#7eade7"))
1048 for idx, item in enumerate(csv_lst[0]):
1049 alignment = "left" if idx == 0 else "center"
1050 th = ET.SubElement(tr, "th", attrib=dict(align=alignment))
1054 colors = ("#e9f1fb", "#d4e4f7")
1055 for r_idx, row in enumerate(csv_lst[1:]):
1056 background = colors[r_idx % 2]
1057 tr = ET.SubElement(failed_tests, "tr", attrib=dict(bgcolor=background))
1060 for c_idx, item in enumerate(row):
1061 alignment = "left" if c_idx == 0 else "center"
1062 td = ET.SubElement(tr, "td", attrib=dict(align=alignment))
1065 url = _generate_url("../trending/", testbed, item)
1066 ref = ET.SubElement(td, "a", attrib=dict(href=url))
1071 with open(table["output-file"], 'w') as html_file:
1072 logging.info(" Writing file: '{0}'".format(table["output-file"]))
1073 html_file.write(".. raw:: html\n\n\t")
1074 html_file.write(ET.tostring(failed_tests))
1075 html_file.write("\n\t<p><br><br></p>\n")
1077 logging.warning("The output file is not defined.")