1 # Copyright (c) 2019 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Algorithms to generate tables.
22 from string import replace
23 from collections import OrderedDict
24 from numpy import nan, isnan
25 from xml.etree import ElementTree as ET
26 from datetime import datetime as dt
27 from datetime import timedelta
29 from utils import mean, stdev, relative_change, classify_anomalies, \
30 convert_csv_to_pretty_txt, relative_change_stdev
33 REGEX_NIC = re.compile(r'\d*ge\dp\d\D*\d*')
36 def generate_tables(spec, data):
37 """Generate all tables specified in the specification file.
39 :param spec: Specification read from the specification file.
40 :param data: Data to process.
41 :type spec: Specification
45 logging.info("Generating the tables ...")
46 for table in spec.tables:
48 eval(table["algorithm"])(table, data)
49 except NameError as err:
50 logging.error("Probably algorithm '{alg}' is not defined: {err}".
51 format(alg=table["algorithm"], err=repr(err)))
55 def table_details(table, input_data):
56 """Generate the table(s) with algorithm: table_detailed_test_results
57 specified in the specification file.
59 :param table: Table to generate.
60 :param input_data: Data to process.
61 :type table: pandas.Series
62 :type input_data: InputData
65 logging.info(" Generating the table {0} ...".
66 format(table.get("title", "")))
69 logging.info(" Creating the data set for the {0} '{1}'.".
70 format(table.get("type", ""), table.get("title", "")))
71 data = input_data.filter_data(table)
73 # Prepare the header of the tables
75 for column in table["columns"]:
76 header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
78 # Generate the data for the table according to the model in the table
80 job = table["data"].keys()[0]
81 build = str(table["data"][job][0])
83 suites = input_data.suites(job, build)
85 logging.error(" No data available. The table will not be generated.")
88 for suite_longname, suite in suites.iteritems():
90 suite_name = suite["name"]
92 for test in data[job][build].keys():
93 if data[job][build][test]["parent"] in suite_name:
95 for column in table["columns"]:
97 col_data = str(data[job][build][test][column["data"].
98 split(" ")[1]]).replace('"', '""')
99 if column["data"].split(" ")[1] in ("conf-history",
101 col_data = replace(col_data, " |br| ", "",
103 col_data = " |prein| {0} |preout| ".\
104 format(col_data[:-5])
105 row_lst.append('"{0}"'.format(col_data))
107 row_lst.append("No data")
108 table_lst.append(row_lst)
110 # Write the data to file
112 file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
113 table["output-file-ext"])
114 logging.info(" Writing file: '{}'".format(file_name))
115 with open(file_name, "w") as file_handler:
116 file_handler.write(",".join(header) + "\n")
117 for item in table_lst:
118 file_handler.write(",".join(item) + "\n")
120 logging.info(" Done.")
123 def table_merged_details(table, input_data):
124 """Generate the table(s) with algorithm: table_merged_details
125 specified in the specification file.
127 :param table: Table to generate.
128 :param input_data: Data to process.
129 :type table: pandas.Series
130 :type input_data: InputData
133 logging.info(" Generating the table {0} ...".
134 format(table.get("title", "")))
137 logging.info(" Creating the data set for the {0} '{1}'.".
138 format(table.get("type", ""), table.get("title", "")))
139 data = input_data.filter_data(table)
140 data = input_data.merge_data(data)
141 data.sort_index(inplace=True)
143 logging.info(" Creating the data set for the {0} '{1}'.".
144 format(table.get("type", ""), table.get("title", "")))
145 suites = input_data.filter_data(table, data_set="suites")
146 suites = input_data.merge_data(suites)
148 # Prepare the header of the tables
150 for column in table["columns"]:
151 header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
153 for _, suite in suites.iteritems():
155 suite_name = suite["name"]
157 for test in data.keys():
158 if data[test]["parent"] in suite_name:
160 for column in table["columns"]:
162 col_data = str(data[test][column["data"].
163 split(" ")[1]]).replace('"', '""')
164 col_data = replace(col_data, "No Data",
166 if column["data"].split(" ")[1] in ("conf-history",
168 col_data = replace(col_data, " |br| ", "",
170 col_data = " |prein| {0} |preout| ".\
171 format(col_data[:-5])
172 row_lst.append('"{0}"'.format(col_data))
174 row_lst.append('"Not captured"')
175 table_lst.append(row_lst)
177 # Write the data to file
179 file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
180 table["output-file-ext"])
181 logging.info(" Writing file: '{}'".format(file_name))
182 with open(file_name, "w") as file_handler:
183 file_handler.write(",".join(header) + "\n")
184 for item in table_lst:
185 file_handler.write(",".join(item) + "\n")
187 logging.info(" Done.")
190 def table_performance_comparison(table, input_data):
191 """Generate the table(s) with algorithm: table_performance_comparison
192 specified in the specification file.
194 :param table: Table to generate.
195 :param input_data: Data to process.
196 :type table: pandas.Series
197 :type input_data: InputData
200 logging.info(" Generating the table {0} ...".
201 format(table.get("title", "")))
204 logging.info(" Creating the data set for the {0} '{1}'.".
205 format(table.get("type", ""), table.get("title", "")))
206 data = input_data.filter_data(table, continue_on_error=True)
208 # Prepare the header of the tables
210 header = ["Test case", ]
212 if table["include-tests"] == "MRR":
213 hdr_param = "Receive Rate"
215 hdr_param = "Throughput"
217 history = table.get("history", None)
221 ["{0} {1} [Mpps]".format(item["title"], hdr_param),
222 "{0} Stdev [Mpps]".format(item["title"])])
224 ["{0} {1} [Mpps]".format(table["reference"]["title"], hdr_param),
225 "{0} Stdev [Mpps]".format(table["reference"]["title"]),
226 "{0} {1} [Mpps]".format(table["compare"]["title"], hdr_param),
227 "{0} Stdev [Mpps]".format(table["compare"]["title"]),
229 header_str = ",".join(header) + "\n"
230 except (AttributeError, KeyError) as err:
231 logging.error("The model is invalid, missing parameter: {0}".
235 # Prepare data to the table:
237 for job, builds in table["reference"]["data"].items():
239 for tst_name, tst_data in data[job][str(build)].iteritems():
240 tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\
241 replace("-ndrpdr", "").replace("-pdrdisc", "").\
242 replace("-ndrdisc", "").replace("-pdr", "").\
243 replace("-ndr", "").\
244 replace("1t1c", "1c").replace("2t1c", "1c").\
245 replace("2t2c", "2c").replace("4t2c", "2c").\
246 replace("4t4c", "4c").replace("8t4c", "4c")
247 if "across topologies" in table["title"].lower():
248 tst_name_mod = tst_name_mod.replace("2n1l-", "")
249 if tbl_dict.get(tst_name_mod, None) is None:
250 groups = re.search(REGEX_NIC, tst_data["parent"])
251 nic = groups.group(0) if groups else ""
252 name = "{0}-{1}".format(nic, "-".join(tst_data["name"].
254 if "across testbeds" in table["title"].lower() or \
255 "across topologies" in table["title"].lower():
257 replace("1t1c", "1c").replace("2t1c", "1c").\
258 replace("2t2c", "2c").replace("4t2c", "2c").\
259 replace("4t4c", "4c").replace("8t4c", "4c")
260 tbl_dict[tst_name_mod] = {"name": name,
264 # TODO: Re-work when NDRPDRDISC tests are not used
265 if table["include-tests"] == "MRR":
266 tbl_dict[tst_name_mod]["ref-data"]. \
267 append(tst_data["result"]["receive-rate"].avg)
268 elif table["include-tests"] == "PDR":
269 if tst_data["type"] == "PDR":
270 tbl_dict[tst_name_mod]["ref-data"]. \
271 append(tst_data["throughput"]["value"])
272 elif tst_data["type"] == "NDRPDR":
273 tbl_dict[tst_name_mod]["ref-data"].append(
274 tst_data["throughput"]["PDR"]["LOWER"])
275 elif table["include-tests"] == "NDR":
276 if tst_data["type"] == "NDR":
277 tbl_dict[tst_name_mod]["ref-data"]. \
278 append(tst_data["throughput"]["value"])
279 elif tst_data["type"] == "NDRPDR":
280 tbl_dict[tst_name_mod]["ref-data"].append(
281 tst_data["throughput"]["NDR"]["LOWER"])
285 pass # No data in output.xml for this test
287 for job, builds in table["compare"]["data"].items():
289 for tst_name, tst_data in data[job][str(build)].iteritems():
290 tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \
291 replace("-ndrpdr", "").replace("-pdrdisc", ""). \
292 replace("-ndrdisc", "").replace("-pdr", ""). \
293 replace("-ndr", "").\
294 replace("1t1c", "1c").replace("2t1c", "1c").\
295 replace("2t2c", "2c").replace("4t2c", "2c").\
296 replace("4t4c", "4c").replace("8t4c", "4c")
297 if "across topologies" in table["title"].lower():
298 tst_name_mod = tst_name_mod.replace("2n1l-", "")
300 # TODO: Re-work when NDRPDRDISC tests are not used
301 if table["include-tests"] == "MRR":
302 tbl_dict[tst_name_mod]["cmp-data"]. \
303 append(tst_data["result"]["receive-rate"].avg)
304 elif table["include-tests"] == "PDR":
305 if tst_data["type"] == "PDR":
306 tbl_dict[tst_name_mod]["cmp-data"]. \
307 append(tst_data["throughput"]["value"])
308 elif tst_data["type"] == "NDRPDR":
309 tbl_dict[tst_name_mod]["cmp-data"].append(
310 tst_data["throughput"]["PDR"]["LOWER"])
311 elif table["include-tests"] == "NDR":
312 if tst_data["type"] == "NDR":
313 tbl_dict[tst_name_mod]["cmp-data"]. \
314 append(tst_data["throughput"]["value"])
315 elif tst_data["type"] == "NDRPDR":
316 tbl_dict[tst_name_mod]["cmp-data"].append(
317 tst_data["throughput"]["NDR"]["LOWER"])
323 tbl_dict.pop(tst_name_mod, None)
326 for job, builds in item["data"].items():
328 for tst_name, tst_data in data[job][str(build)].iteritems():
329 tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \
330 replace("-ndrpdr", "").replace("-pdrdisc", ""). \
331 replace("-ndrdisc", "").replace("-pdr", ""). \
332 replace("-ndr", "").\
333 replace("1t1c", "1c").replace("2t1c", "1c").\
334 replace("2t2c", "2c").replace("4t2c", "2c").\
335 replace("4t4c", "4c").replace("8t4c", "4c")
336 if "across topologies" in table["title"].lower():
337 tst_name_mod = tst_name_mod.replace("2n1l-", "")
338 if tbl_dict.get(tst_name_mod, None) is None:
340 if tbl_dict[tst_name_mod].get("history", None) is None:
341 tbl_dict[tst_name_mod]["history"] = OrderedDict()
342 if tbl_dict[tst_name_mod]["history"].get(item["title"],
344 tbl_dict[tst_name_mod]["history"][item["title"]] = \
347 # TODO: Re-work when NDRPDRDISC tests are not used
348 if table["include-tests"] == "MRR":
349 tbl_dict[tst_name_mod]["history"][item["title"
350 ]].append(tst_data["result"]["receive-rate"].
352 elif table["include-tests"] == "PDR":
353 if tst_data["type"] == "PDR":
354 tbl_dict[tst_name_mod]["history"][
356 append(tst_data["throughput"]["value"])
357 elif tst_data["type"] == "NDRPDR":
358 tbl_dict[tst_name_mod]["history"][item[
359 "title"]].append(tst_data["throughput"][
361 elif table["include-tests"] == "NDR":
362 if tst_data["type"] == "NDR":
363 tbl_dict[tst_name_mod]["history"][
365 append(tst_data["throughput"]["value"])
366 elif tst_data["type"] == "NDRPDR":
367 tbl_dict[tst_name_mod]["history"][item[
368 "title"]].append(tst_data["throughput"][
372 except (TypeError, KeyError):
376 for tst_name in tbl_dict.keys():
377 item = [tbl_dict[tst_name]["name"], ]
379 if tbl_dict[tst_name].get("history", None) is not None:
380 for hist_data in tbl_dict[tst_name]["history"].values():
382 item.append(round(mean(hist_data) / 1000000, 2))
383 item.append(round(stdev(hist_data) / 1000000, 2))
385 item.extend([None, None])
387 item.extend([None, None])
388 data_t = tbl_dict[tst_name]["ref-data"]
390 item.append(round(mean(data_t) / 1000000, 2))
391 item.append(round(stdev(data_t) / 1000000, 2))
393 item.extend([None, None])
394 data_t = tbl_dict[tst_name]["cmp-data"]
396 item.append(round(mean(data_t) / 1000000, 2))
397 item.append(round(stdev(data_t) / 1000000, 2))
399 item.extend([None, None])
400 if item[-4] is not None and item[-2] is not None and item[-4] != 0:
401 item.append(int(relative_change(float(item[-4]), float(item[-2]))))
402 if len(item) == len(header):
405 # Sort the table according to the relative change
406 tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
408 # Generate csv tables:
409 csv_file = "{0}.csv".format(table["output-file"])
410 with open(csv_file, "w") as file_handler:
411 file_handler.write(header_str)
413 file_handler.write(",".join([str(item) for item in test]) + "\n")
415 convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
418 def table_performance_comparison_nic(table, input_data):
419 """Generate the table(s) with algorithm: table_performance_comparison
420 specified in the specification file.
422 :param table: Table to generate.
423 :param input_data: Data to process.
424 :type table: pandas.Series
425 :type input_data: InputData
428 logging.info(" Generating the table {0} ...".
429 format(table.get("title", "")))
432 logging.info(" Creating the data set for the {0} '{1}'.".
433 format(table.get("type", ""), table.get("title", "")))
434 data = input_data.filter_data(table, continue_on_error=True)
436 # Prepare the header of the tables
438 header = ["Test case", ]
440 if table["include-tests"] == "MRR":
441 hdr_param = "Receive Rate"
443 hdr_param = "Throughput"
445 history = table.get("history", None)
449 ["{0} {1} [Mpps]".format(item["title"], hdr_param),
450 "{0} Stdev [Mpps]".format(item["title"])])
452 ["{0} {1} [Mpps]".format(table["reference"]["title"], hdr_param),
453 "{0} Stdev [Mpps]".format(table["reference"]["title"]),
454 "{0} {1} [Mpps]".format(table["compare"]["title"], hdr_param),
455 "{0} Stdev [Mpps]".format(table["compare"]["title"]),
457 header_str = ",".join(header) + "\n"
458 except (AttributeError, KeyError) as err:
459 logging.error("The model is invalid, missing parameter: {0}".
463 # Prepare data to the table:
465 for job, builds in table["reference"]["data"].items():
467 for tst_name, tst_data in data[job][str(build)].iteritems():
468 if table["reference"]["nic"] not in tst_data["tags"]:
470 tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\
471 replace("-ndrpdr", "").replace("-pdrdisc", "").\
472 replace("-ndrdisc", "").replace("-pdr", "").\
473 replace("-ndr", "").\
474 replace("1t1c", "1c").replace("2t1c", "1c").\
475 replace("2t2c", "2c").replace("4t2c", "2c").\
476 replace("4t4c", "4c").replace("8t4c", "4c")
477 tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod)
478 if "across topologies" in table["title"].lower():
479 tst_name_mod = tst_name_mod.replace("2n1l-", "")
480 if tbl_dict.get(tst_name_mod, None) is None:
481 name = "{0}".format("-".join(tst_data["name"].
483 if "across testbeds" in table["title"].lower() or \
484 "across topologies" in table["title"].lower():
486 replace("1t1c", "1c").replace("2t1c", "1c").\
487 replace("2t2c", "2c").replace("4t2c", "2c").\
488 replace("4t4c", "4c").replace("8t4c", "4c")
489 tbl_dict[tst_name_mod] = {"name": name,
493 # TODO: Re-work when NDRPDRDISC tests are not used
494 if table["include-tests"] == "MRR":
495 tbl_dict[tst_name_mod]["ref-data"]. \
496 append(tst_data["result"]["receive-rate"].avg)
497 elif table["include-tests"] == "PDR":
498 if tst_data["type"] == "PDR":
499 tbl_dict[tst_name_mod]["ref-data"]. \
500 append(tst_data["throughput"]["value"])
501 elif tst_data["type"] == "NDRPDR":
502 tbl_dict[tst_name_mod]["ref-data"].append(
503 tst_data["throughput"]["PDR"]["LOWER"])
504 elif table["include-tests"] == "NDR":
505 if tst_data["type"] == "NDR":
506 tbl_dict[tst_name_mod]["ref-data"]. \
507 append(tst_data["throughput"]["value"])
508 elif tst_data["type"] == "NDRPDR":
509 tbl_dict[tst_name_mod]["ref-data"].append(
510 tst_data["throughput"]["NDR"]["LOWER"])
514 pass # No data in output.xml for this test
516 for job, builds in table["compare"]["data"].items():
518 for tst_name, tst_data in data[job][str(build)].iteritems():
519 if table["compare"]["nic"] not in tst_data["tags"]:
521 tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \
522 replace("-ndrpdr", "").replace("-pdrdisc", ""). \
523 replace("-ndrdisc", "").replace("-pdr", ""). \
524 replace("-ndr", "").\
525 replace("1t1c", "1c").replace("2t1c", "1c").\
526 replace("2t2c", "2c").replace("4t2c", "2c").\
527 replace("4t4c", "4c").replace("8t4c", "4c")
528 tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod)
529 if "across topologies" in table["title"].lower():
530 tst_name_mod = tst_name_mod.replace("2n1l-", "")
531 if tbl_dict.get(tst_name_mod, None) is None:
532 name = "{0}".format("-".join(tst_data["name"].
534 if "across testbeds" in table["title"].lower() or \
535 "across topologies" in table["title"].lower():
537 replace("1t1c", "1c").replace("2t1c", "1c").\
538 replace("2t2c", "2c").replace("4t2c", "2c").\
539 replace("4t4c", "4c").replace("8t4c", "4c")
540 tbl_dict[tst_name_mod] = {"name": name,
544 # TODO: Re-work when NDRPDRDISC tests are not used
545 if table["include-tests"] == "MRR":
546 tbl_dict[tst_name_mod]["cmp-data"]. \
547 append(tst_data["result"]["receive-rate"].avg)
548 elif table["include-tests"] == "PDR":
549 if tst_data["type"] == "PDR":
550 tbl_dict[tst_name_mod]["cmp-data"]. \
551 append(tst_data["throughput"]["value"])
552 elif tst_data["type"] == "NDRPDR":
553 tbl_dict[tst_name_mod]["cmp-data"].append(
554 tst_data["throughput"]["PDR"]["LOWER"])
555 elif table["include-tests"] == "NDR":
556 if tst_data["type"] == "NDR":
557 tbl_dict[tst_name_mod]["cmp-data"]. \
558 append(tst_data["throughput"]["value"])
559 elif tst_data["type"] == "NDRPDR":
560 tbl_dict[tst_name_mod]["cmp-data"].append(
561 tst_data["throughput"]["NDR"]["LOWER"])
564 except (KeyError, TypeError):
569 for job, builds in item["data"].items():
571 for tst_name, tst_data in data[job][str(build)].iteritems():
572 if item["nic"] not in tst_data["tags"]:
574 tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \
575 replace("-ndrpdr", "").replace("-pdrdisc", ""). \
576 replace("-ndrdisc", "").replace("-pdr", ""). \
577 replace("-ndr", "").\
578 replace("1t1c", "1c").replace("2t1c", "1c").\
579 replace("2t2c", "2c").replace("4t2c", "2c").\
580 replace("4t4c", "4c").replace("8t4c", "4c")
581 tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod)
582 if "across topologies" in table["title"].lower():
583 tst_name_mod = tst_name_mod.replace("2n1l-", "")
584 if tbl_dict.get(tst_name_mod, None) is None:
586 if tbl_dict[tst_name_mod].get("history", None) is None:
587 tbl_dict[tst_name_mod]["history"] = OrderedDict()
588 if tbl_dict[tst_name_mod]["history"].get(item["title"],
590 tbl_dict[tst_name_mod]["history"][item["title"]] = \
593 # TODO: Re-work when NDRPDRDISC tests are not used
594 if table["include-tests"] == "MRR":
595 tbl_dict[tst_name_mod]["history"][item["title"
596 ]].append(tst_data["result"]["receive-rate"].
598 elif table["include-tests"] == "PDR":
599 if tst_data["type"] == "PDR":
600 tbl_dict[tst_name_mod]["history"][
602 append(tst_data["throughput"]["value"])
603 elif tst_data["type"] == "NDRPDR":
604 tbl_dict[tst_name_mod]["history"][item[
605 "title"]].append(tst_data["throughput"][
607 elif table["include-tests"] == "NDR":
608 if tst_data["type"] == "NDR":
609 tbl_dict[tst_name_mod]["history"][
611 append(tst_data["throughput"]["value"])
612 elif tst_data["type"] == "NDRPDR":
613 tbl_dict[tst_name_mod]["history"][item[
614 "title"]].append(tst_data["throughput"][
618 except (TypeError, KeyError):
622 for tst_name in tbl_dict.keys():
623 item = [tbl_dict[tst_name]["name"], ]
625 if tbl_dict[tst_name].get("history", None) is not None:
626 for hist_data in tbl_dict[tst_name]["history"].values():
628 item.append(round(mean(hist_data) / 1000000, 2))
629 item.append(round(stdev(hist_data) / 1000000, 2))
631 item.extend([None, None])
633 item.extend([None, None])
634 data_t = tbl_dict[tst_name]["ref-data"]
636 item.append(round(mean(data_t) / 1000000, 2))
637 item.append(round(stdev(data_t) / 1000000, 2))
639 item.extend([None, None])
640 data_t = tbl_dict[tst_name]["cmp-data"]
642 item.append(round(mean(data_t) / 1000000, 2))
643 item.append(round(stdev(data_t) / 1000000, 2))
645 item.extend([None, None])
646 if item[-4] is not None and item[-2] is not None and item[-4] != 0:
647 item.append(int(relative_change(float(item[-4]), float(item[-2]))))
650 if len(item) == len(header):
653 # Sort the table according to the relative change
654 tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
656 # Generate csv tables:
657 csv_file = "{0}.csv".format(table["output-file"])
658 with open(csv_file, "w") as file_handler:
659 file_handler.write(header_str)
661 file_handler.write(",".join([str(item) for item in test]) + "\n")
663 convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
666 def table_nics_comparison(table, input_data):
667 """Generate the table(s) with algorithm: table_nics_comparison
668 specified in the specification file.
670 :param table: Table to generate.
671 :param input_data: Data to process.
672 :type table: pandas.Series
673 :type input_data: InputData
676 logging.info(" Generating the table {0} ...".
677 format(table.get("title", "")))
680 logging.info(" Creating the data set for the {0} '{1}'.".
681 format(table.get("type", ""), table.get("title", "")))
682 data = input_data.filter_data(table, continue_on_error=True)
684 # Prepare the header of the tables
686 header = ["Test case", ]
688 if table["include-tests"] == "MRR":
689 hdr_param = "Receive Rate"
691 hdr_param = "Throughput"
694 ["{0} {1} [Mpps]".format(table["reference"]["title"], hdr_param),
695 "{0} Stdev [Mpps]".format(table["reference"]["title"]),
696 "{0} {1} [Mpps]".format(table["compare"]["title"], hdr_param),
697 "{0} Stdev [Mpps]".format(table["compare"]["title"]),
699 header_str = ",".join(header) + "\n"
700 except (AttributeError, KeyError) as err:
701 logging.error("The model is invalid, missing parameter: {0}".
705 # Prepare data to the table:
707 for job, builds in table["data"].items():
709 for tst_name, tst_data in data[job][str(build)].iteritems():
710 tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\
711 replace("-ndrpdr", "").replace("-pdrdisc", "").\
712 replace("-ndrdisc", "").replace("-pdr", "").\
713 replace("-ndr", "").\
714 replace("1t1c", "1c").replace("2t1c", "1c").\
715 replace("2t2c", "2c").replace("4t2c", "2c").\
716 replace("4t4c", "4c").replace("8t4c", "4c")
717 tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod)
718 if tbl_dict.get(tst_name_mod, None) is None:
719 name = "-".join(tst_data["name"].split("-")[:-1])
720 tbl_dict[tst_name_mod] = {"name": name,
724 if table["include-tests"] == "MRR":
725 result = tst_data["result"]["receive-rate"].avg
726 elif table["include-tests"] == "PDR":
727 result = tst_data["throughput"]["PDR"]["LOWER"]
728 elif table["include-tests"] == "NDR":
729 result = tst_data["throughput"]["NDR"]["LOWER"]
734 if table["reference"]["nic"] in tst_data["tags"]:
735 tbl_dict[tst_name_mod]["ref-data"].append(result)
736 elif table["compare"]["nic"] in tst_data["tags"]:
737 tbl_dict[tst_name_mod]["cmp-data"].append(result)
738 except (TypeError, KeyError) as err:
739 logging.debug("No data for {0}".format(tst_name))
740 logging.debug(repr(err))
741 # No data in output.xml for this test
744 for tst_name in tbl_dict.keys():
745 item = [tbl_dict[tst_name]["name"], ]
746 data_t = tbl_dict[tst_name]["ref-data"]
748 item.append(round(mean(data_t) / 1000000, 2))
749 item.append(round(stdev(data_t) / 1000000, 2))
751 item.extend([None, None])
752 data_t = tbl_dict[tst_name]["cmp-data"]
754 item.append(round(mean(data_t) / 1000000, 2))
755 item.append(round(stdev(data_t) / 1000000, 2))
757 item.extend([None, None])
758 if item[-4] is not None and item[-2] is not None and item[-4] != 0:
759 item.append(int(relative_change(float(item[-4]), float(item[-2]))))
760 if len(item) == len(header):
763 # Sort the table according to the relative change
764 tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
766 # Generate csv tables:
767 csv_file = "{0}.csv".format(table["output-file"])
768 with open(csv_file, "w") as file_handler:
769 file_handler.write(header_str)
771 file_handler.write(",".join([str(item) for item in test]) + "\n")
773 convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
776 def table_soak_vs_ndr(table, input_data):
777 """Generate the table(s) with algorithm: table_soak_vs_ndr
778 specified in the specification file.
780 :param table: Table to generate.
781 :param input_data: Data to process.
782 :type table: pandas.Series
783 :type input_data: InputData
786 logging.info(" Generating the table {0} ...".
787 format(table.get("title", "")))
790 logging.info(" Creating the data set for the {0} '{1}'.".
791 format(table.get("type", ""), table.get("title", "")))
792 data = input_data.filter_data(table, continue_on_error=True)
794 # Prepare the header of the table
798 "{0} Throughput [Mpps]".format(table["reference"]["title"]),
799 "{0} Stdev [Mpps]".format(table["reference"]["title"]),
800 "{0} Throughput [Mpps]".format(table["compare"]["title"]),
801 "{0} Stdev [Mpps]".format(table["compare"]["title"]),
802 "Delta [%]", "Stdev of delta [%]"]
803 header_str = ",".join(header) + "\n"
804 except (AttributeError, KeyError) as err:
805 logging.error("The model is invalid, missing parameter: {0}".
809 # Create a list of available SOAK test results:
811 for job, builds in table["compare"]["data"].items():
813 for tst_name, tst_data in data[job][str(build)].iteritems():
814 if tst_data["type"] == "SOAK":
815 tst_name_mod = tst_name.replace("-soak", "")
816 if tbl_dict.get(tst_name_mod, None) is None:
817 groups = re.search(REGEX_NIC, tst_data["parent"])
818 nic = groups.group(0) if groups else ""
819 name = "{0}-{1}".format(nic, "-".join(tst_data["name"].
821 tbl_dict[tst_name_mod] = {
827 tbl_dict[tst_name_mod]["cmp-data"].append(
828 tst_data["throughput"]["LOWER"])
829 except (KeyError, TypeError):
831 tests_lst = tbl_dict.keys()
833 # Add corresponding NDR test results:
834 for job, builds in table["reference"]["data"].items():
836 for tst_name, tst_data in data[job][str(build)].iteritems():
837 tst_name_mod = tst_name.replace("-ndrpdr", "").\
839 if tst_name_mod in tests_lst:
841 if tst_data["type"] in ("NDRPDR", "MRR", "BMRR"):
842 if table["include-tests"] == "MRR":
843 result = tst_data["result"]["receive-rate"].avg
844 elif table["include-tests"] == "PDR":
845 result = tst_data["throughput"]["PDR"]["LOWER"]
846 elif table["include-tests"] == "NDR":
847 result = tst_data["throughput"]["NDR"]["LOWER"]
850 if result is not None:
851 tbl_dict[tst_name_mod]["ref-data"].append(
853 except (KeyError, TypeError):
857 for tst_name in tbl_dict.keys():
858 item = [tbl_dict[tst_name]["name"], ]
859 data_r = tbl_dict[tst_name]["ref-data"]
861 data_r_mean = mean(data_r)
862 item.append(round(data_r_mean / 1000000, 2))
863 data_r_stdev = stdev(data_r)
864 item.append(round(data_r_stdev / 1000000, 2))
868 item.extend([None, None])
869 data_c = tbl_dict[tst_name]["cmp-data"]
871 data_c_mean = mean(data_c)
872 item.append(round(data_c_mean / 1000000, 2))
873 data_c_stdev = stdev(data_c)
874 item.append(round(data_c_stdev / 1000000, 2))
878 item.extend([None, None])
879 if data_r_mean and data_c_mean:
880 delta, d_stdev = relative_change_stdev(
881 data_r_mean, data_c_mean, data_r_stdev, data_c_stdev)
882 item.append(round(delta, 2))
883 item.append(round(d_stdev, 2))
886 # Sort the table according to the relative change
887 tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
889 # Generate csv tables:
890 csv_file = "{0}.csv".format(table["output-file"])
891 with open(csv_file, "w") as file_handler:
892 file_handler.write(header_str)
894 file_handler.write(",".join([str(item) for item in test]) + "\n")
896 convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
899 def table_performance_trending_dashboard(table, input_data):
900 """Generate the table(s) with algorithm:
901 table_performance_trending_dashboard
902 specified in the specification file.
904 :param table: Table to generate.
905 :param input_data: Data to process.
906 :type table: pandas.Series
907 :type input_data: InputData
910 logging.info(" Generating the table {0} ...".
911 format(table.get("title", "")))
914 logging.info(" Creating the data set for the {0} '{1}'.".
915 format(table.get("type", ""), table.get("title", "")))
916 data = input_data.filter_data(table, continue_on_error=True)
918 # Prepare the header of the tables
919 header = ["Test Case",
921 "Short-Term Change [%]",
922 "Long-Term Change [%]",
926 header_str = ",".join(header) + "\n"
928 # Prepare data to the table:
930 for job, builds in table["data"].items():
932 for tst_name, tst_data in data[job][str(build)].iteritems():
933 if tst_name.lower() in table.get("ignore-list", list()):
935 if tbl_dict.get(tst_name, None) is None:
936 groups = re.search(REGEX_NIC, tst_data["parent"])
939 nic = groups.group(0)
940 tbl_dict[tst_name] = {
941 "name": "{0}-{1}".format(nic, tst_data["name"]),
942 "data": OrderedDict()}
944 tbl_dict[tst_name]["data"][str(build)] = \
945 tst_data["result"]["receive-rate"]
946 except (TypeError, KeyError):
947 pass # No data in output.xml for this test
950 for tst_name in tbl_dict.keys():
951 data_t = tbl_dict[tst_name]["data"]
955 classification_lst, avgs = classify_anomalies(data_t)
957 win_size = min(len(data_t), table["window"])
958 long_win_size = min(len(data_t), table["long-trend-window"])
962 [x for x in avgs[-long_win_size:-win_size]
967 avg_week_ago = avgs[max(-win_size, -len(avgs))]
969 if isnan(last_avg) or isnan(avg_week_ago) or avg_week_ago == 0.0:
970 rel_change_last = nan
972 rel_change_last = round(
973 ((last_avg - avg_week_ago) / avg_week_ago) * 100, 2)
975 if isnan(max_long_avg) or isnan(last_avg) or max_long_avg == 0.0:
976 rel_change_long = nan
978 rel_change_long = round(
979 ((last_avg - max_long_avg) / max_long_avg) * 100, 2)
981 if classification_lst:
982 if isnan(rel_change_last) and isnan(rel_change_long):
984 if (isnan(last_avg) or
985 isnan(rel_change_last) or
986 isnan(rel_change_long)):
989 [tbl_dict[tst_name]["name"],
990 round(last_avg / 1000000, 2),
993 classification_lst[-win_size:].count("regression"),
994 classification_lst[-win_size:].count("progression")])
996 tbl_lst.sort(key=lambda rel: rel[0])
999 for nrr in range(table["window"], -1, -1):
1000 tbl_reg = [item for item in tbl_lst if item[4] == nrr]
1001 for nrp in range(table["window"], -1, -1):
1002 tbl_out = [item for item in tbl_reg if item[5] == nrp]
1003 tbl_out.sort(key=lambda rel: rel[2])
1004 tbl_sorted.extend(tbl_out)
1006 file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
1008 logging.info(" Writing file: '{0}'".format(file_name))
1009 with open(file_name, "w") as file_handler:
1010 file_handler.write(header_str)
1011 for test in tbl_sorted:
1012 file_handler.write(",".join([str(item) for item in test]) + '\n')
1014 txt_file_name = "{0}.txt".format(table["output-file"])
1015 logging.info(" Writing file: '{0}'".format(txt_file_name))
1016 convert_csv_to_pretty_txt(file_name, txt_file_name)
1019 def _generate_url(base, testbed, test_name):
1020 """Generate URL to a trending plot from the name of the test case.
1022 :param base: The base part of URL common to all test cases.
1023 :param testbed: The testbed used for testing.
1024 :param test_name: The name of the test case.
1027 :type test_name: str
1028 :returns: The URL to the plot with the trending data for the given test
1038 if "lbdpdk" in test_name or "lbvpp" in test_name:
1039 file_name = "link_bonding"
1041 elif "114b" in test_name and "vhost" in test_name:
1044 elif "testpmd" in test_name or "l3fwd" in test_name:
1047 elif "memif" in test_name:
1048 file_name = "container_memif"
1051 elif "srv6" in test_name:
1054 elif "vhost" in test_name:
1055 if "l2xcbase" in test_name or "l2bdbasemaclrn" in test_name:
1056 file_name = "vm_vhost_l2"
1057 if "114b" in test_name:
1059 elif "l2xcbase" in test_name and "x520" in test_name:
1060 feature = "-base-l2xc"
1061 elif "l2bdbasemaclrn" in test_name and "x520" in test_name:
1062 feature = "-base-l2bd"
1065 elif "ip4base" in test_name:
1066 file_name = "vm_vhost_ip4"
1069 elif "ipsecbasetnlsw" in test_name:
1070 file_name = "ipsecsw"
1071 feature = "-base-scale"
1073 elif "ipsec" in test_name:
1075 feature = "-base-scale"
1076 if "hw-" in test_name:
1077 file_name = "ipsechw"
1078 elif "sw-" in test_name:
1079 file_name = "ipsecsw"
1081 elif "ethip4lispip" in test_name or "ethip4vxlan" in test_name:
1082 file_name = "ip4_tunnels"
1085 elif "ip4base" in test_name or "ip4scale" in test_name:
1087 if "xl710" in test_name:
1088 feature = "-base-scale-features"
1089 elif "iacl" in test_name:
1090 feature = "-features-iacl"
1091 elif "oacl" in test_name:
1092 feature = "-features-oacl"
1093 elif "snat" in test_name or "cop" in test_name:
1094 feature = "-features"
1096 feature = "-base-scale"
1098 elif "ip6base" in test_name or "ip6scale" in test_name:
1100 feature = "-base-scale"
1102 elif "l2xcbase" in test_name or "l2xcscale" in test_name \
1103 or "l2bdbasemaclrn" in test_name or "l2bdscale" in test_name \
1104 or "l2dbbasemaclrn" in test_name or "l2dbscale" in test_name:
1106 if "macip" in test_name:
1107 feature = "-features-macip"
1108 elif "iacl" in test_name:
1109 feature = "-features-iacl"
1110 elif "oacl" in test_name:
1111 feature = "-features-oacl"
1113 feature = "-base-scale"
1115 if "x520" in test_name:
1117 elif "x710" in test_name:
1119 elif "xl710" in test_name:
1121 elif "xxv710" in test_name:
1123 elif "vic1227" in test_name:
1125 elif "vic1385" in test_name:
1131 if "64b" in test_name:
1133 elif "78b" in test_name:
1135 elif "imix" in test_name:
1137 elif "9000b" in test_name:
1139 elif "1518b" in test_name:
1141 elif "114b" in test_name:
1145 anchor += framesize + '-'
1147 if "1t1c" in test_name:
1149 elif "2t2c" in test_name:
1151 elif "4t4c" in test_name:
1153 elif "2t1c" in test_name:
1155 elif "4t2c" in test_name:
1157 elif "8t4c" in test_name:
1160 return url + file_name + '-' + testbed + '-' + nic + framesize + \
1161 feature.replace("-int", "").replace("-tnl", "") + anchor + feature
1164 def table_performance_trending_dashboard_html(table, input_data):
1165 """Generate the table(s) with algorithm:
1166 table_performance_trending_dashboard_html specified in the specification
1169 :param table: Table to generate.
1170 :param input_data: Data to process.
1172 :type input_data: InputData
1175 testbed = table.get("testbed", None)
1177 logging.error("The testbed is not defined for the table '{0}'.".
1178 format(table.get("title", "")))
1181 logging.info(" Generating the table {0} ...".
1182 format(table.get("title", "")))
1185 with open(table["input-file"], 'rb') as csv_file:
1186 csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
1187 csv_lst = [item for item in csv_content]
1189 logging.warning("The input file is not defined.")
1191 except csv.Error as err:
1192 logging.warning("Not possible to process the file '{0}'.\n{1}".
1193 format(table["input-file"], err))
1197 dashboard = ET.Element("table", attrib=dict(width="100%", border='0'))
1200 tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor="#7eade7"))
1201 for idx, item in enumerate(csv_lst[0]):
1202 alignment = "left" if idx == 0 else "center"
1203 th = ET.SubElement(tr, "th", attrib=dict(align=alignment))
1207 colors = {"regression": ("#ffcccc", "#ff9999"),
1208 "progression": ("#c6ecc6", "#9fdf9f"),
1209 "normal": ("#e9f1fb", "#d4e4f7")}
1210 for r_idx, row in enumerate(csv_lst[1:]):
1212 color = "regression"
1214 color = "progression"
1217 background = colors[color][r_idx % 2]
1218 tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor=background))
1221 for c_idx, item in enumerate(row):
1222 alignment = "left" if c_idx == 0 else "center"
1223 td = ET.SubElement(tr, "td", attrib=dict(align=alignment))
1226 url = _generate_url("../trending/", testbed, item)
1227 ref = ET.SubElement(td, "a", attrib=dict(href=url))
1232 with open(table["output-file"], 'w') as html_file:
1233 logging.info(" Writing file: '{0}'".format(table["output-file"]))
1234 html_file.write(".. raw:: html\n\n\t")
1235 html_file.write(ET.tostring(dashboard))
1236 html_file.write("\n\t<p><br><br></p>\n")
1238 logging.warning("The output file is not defined.")
1242 def table_last_failed_tests(table, input_data):
1243 """Generate the table(s) with algorithm: table_last_failed_tests
1244 specified in the specification file.
1246 :param table: Table to generate.
1247 :param input_data: Data to process.
1248 :type table: pandas.Series
1249 :type input_data: InputData
1252 logging.info(" Generating the table {0} ...".
1253 format(table.get("title", "")))
1255 # Transform the data
1256 logging.info(" Creating the data set for the {0} '{1}'.".
1257 format(table.get("type", ""), table.get("title", "")))
1258 data = input_data.filter_data(table, continue_on_error=True)
1260 if data is None or data.empty:
1261 logging.warn(" No data for the {0} '{1}'.".
1262 format(table.get("type", ""), table.get("title", "")))
1266 for job, builds in table["data"].items():
1267 for build in builds:
1270 version = input_data.metadata(job, build).get("version", "")
1272 logging.error("Data for {job}: {build} is not present.".
1273 format(job=job, build=build))
1275 tbl_list.append(build)
1276 tbl_list.append(version)
1277 for tst_name, tst_data in data[job][build].iteritems():
1278 if tst_data["status"] != "FAIL":
1280 groups = re.search(REGEX_NIC, tst_data["parent"])
1283 nic = groups.group(0)
1284 tbl_list.append("{0}-{1}".format(nic, tst_data["name"]))
1286 file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
1287 logging.info(" Writing file: '{0}'".format(file_name))
1288 with open(file_name, "w") as file_handler:
1289 for test in tbl_list:
1290 file_handler.write(test + '\n')
1293 def table_failed_tests(table, input_data):
1294 """Generate the table(s) with algorithm: table_failed_tests
1295 specified in the specification file.
1297 :param table: Table to generate.
1298 :param input_data: Data to process.
1299 :type table: pandas.Series
1300 :type input_data: InputData
1303 logging.info(" Generating the table {0} ...".
1304 format(table.get("title", "")))
1306 # Transform the data
1307 logging.info(" Creating the data set for the {0} '{1}'.".
1308 format(table.get("type", ""), table.get("title", "")))
1309 data = input_data.filter_data(table, continue_on_error=True)
1311 # Prepare the header of the tables
1312 header = ["Test Case",
1314 "Last Failure [Time]",
1315 "Last Failure [VPP-Build-Id]",
1316 "Last Failure [CSIT-Job-Build-Id]"]
1318 # Generate the data for the table according to the model in the table
1322 timeperiod = timedelta(int(table.get("window", 7)))
1325 for job, builds in table["data"].items():
1326 for build in builds:
1328 for tst_name, tst_data in data[job][build].iteritems():
1329 if tst_name.lower() in table.get("ignore-list", list()):
1331 if tbl_dict.get(tst_name, None) is None:
1332 groups = re.search(REGEX_NIC, tst_data["parent"])
1335 nic = groups.group(0)
1336 tbl_dict[tst_name] = {
1337 "name": "{0}-{1}".format(nic, tst_data["name"]),
1338 "data": OrderedDict()}
1340 generated = input_data.metadata(job, build).\
1341 get("generated", "")
1344 then = dt.strptime(generated, "%Y%m%d %H:%M")
1345 if (now - then) <= timeperiod:
1346 tbl_dict[tst_name]["data"][build] = (
1349 input_data.metadata(job, build).get("version", ""),
1351 except (TypeError, KeyError) as err:
1352 logging.warning("tst_name: {} - err: {}".
1353 format(tst_name, repr(err)))
1357 for tst_data in tbl_dict.values():
1359 for val in tst_data["data"].values():
1360 if val[0] == "FAIL":
1362 fails_last_date = val[1]
1363 fails_last_vpp = val[2]
1364 fails_last_csit = val[3]
1366 max_fails = fails_nr if fails_nr > max_fails else max_fails
1367 tbl_lst.append([tst_data["name"],
1371 "mrr-daily-build-{0}".format(fails_last_csit)])
1373 tbl_lst.sort(key=lambda rel: rel[2], reverse=True)
1375 for nrf in range(max_fails, -1, -1):
1376 tbl_fails = [item for item in tbl_lst if item[1] == nrf]
1377 tbl_sorted.extend(tbl_fails)
1378 file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
1380 logging.info(" Writing file: '{0}'".format(file_name))
1381 with open(file_name, "w") as file_handler:
1382 file_handler.write(",".join(header) + "\n")
1383 for test in tbl_sorted:
1384 file_handler.write(",".join([str(item) for item in test]) + '\n')
1386 txt_file_name = "{0}.txt".format(table["output-file"])
1387 logging.info(" Writing file: '{0}'".format(txt_file_name))
1388 convert_csv_to_pretty_txt(file_name, txt_file_name)
1391 def table_failed_tests_html(table, input_data):
1392 """Generate the table(s) with algorithm: table_failed_tests_html
1393 specified in the specification file.
1395 :param table: Table to generate.
1396 :param input_data: Data to process.
1397 :type table: pandas.Series
1398 :type input_data: InputData
1401 testbed = table.get("testbed", None)
1403 logging.error("The testbed is not defined for the table '{0}'.".
1404 format(table.get("title", "")))
1407 logging.info(" Generating the table {0} ...".
1408 format(table.get("title", "")))
1411 with open(table["input-file"], 'rb') as csv_file:
1412 csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
1413 csv_lst = [item for item in csv_content]
1415 logging.warning("The input file is not defined.")
1417 except csv.Error as err:
1418 logging.warning("Not possible to process the file '{0}'.\n{1}".
1419 format(table["input-file"], err))
1423 failed_tests = ET.Element("table", attrib=dict(width="100%", border='0'))
1426 tr = ET.SubElement(failed_tests, "tr", attrib=dict(bgcolor="#7eade7"))
1427 for idx, item in enumerate(csv_lst[0]):
1428 alignment = "left" if idx == 0 else "center"
1429 th = ET.SubElement(tr, "th", attrib=dict(align=alignment))
1433 colors = ("#e9f1fb", "#d4e4f7")
1434 for r_idx, row in enumerate(csv_lst[1:]):
1435 background = colors[r_idx % 2]
1436 tr = ET.SubElement(failed_tests, "tr", attrib=dict(bgcolor=background))
1439 for c_idx, item in enumerate(row):
1440 alignment = "left" if c_idx == 0 else "center"
1441 td = ET.SubElement(tr, "td", attrib=dict(align=alignment))
1444 url = _generate_url("../trending/", testbed, item)
1445 ref = ET.SubElement(td, "a", attrib=dict(href=url))
1450 with open(table["output-file"], 'w') as html_file:
1451 logging.info(" Writing file: '{0}'".format(table["output-file"]))
1452 html_file.write(".. raw:: html\n\n\t")
1453 html_file.write(ET.tostring(failed_tests))
1454 html_file.write("\n\t<p><br><br></p>\n")
1456 logging.warning("The output file is not defined.")