CSIT-1504: Soak tests - box plots
[csit.git] / resources / tools / presentation / generator_tables.py
1 # Copyright (c) 2019 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
5 #
6 #     http://www.apache.org/licenses/LICENSE-2.0
7 #
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
13
14 """Algorithms to generate tables.
15 """
16
17
18 import logging
19 import csv
20 import re
21
22 from string import replace
23 from collections import OrderedDict
24 from numpy import nan, isnan
25 from xml.etree import ElementTree as ET
26 from datetime import datetime as dt
27 from datetime import timedelta
28
29 from utils import mean, stdev, relative_change, classify_anomalies, \
30     convert_csv_to_pretty_txt
31
32
33 REGEX_NIC = re.compile(r'\d*ge\dp\d\D*\d*')
34
35
36 def generate_tables(spec, data):
37     """Generate all tables specified in the specification file.
38
39     :param spec: Specification read from the specification file.
40     :param data: Data to process.
41     :type spec: Specification
42     :type data: InputData
43     """
44
45     logging.info("Generating the tables ...")
46     for table in spec.tables:
47         try:
48             eval(table["algorithm"])(table, data)
49         except NameError as err:
50             logging.error("Probably algorithm '{alg}' is not defined: {err}".
51                           format(alg=table["algorithm"], err=repr(err)))
52     logging.info("Done.")
53
54
55 def table_details(table, input_data):
56     """Generate the table(s) with algorithm: table_detailed_test_results
57     specified in the specification file.
58
59     :param table: Table to generate.
60     :param input_data: Data to process.
61     :type table: pandas.Series
62     :type input_data: InputData
63     """
64
65     logging.info("  Generating the table {0} ...".
66                  format(table.get("title", "")))
67
68     # Transform the data
69     logging.info("    Creating the data set for the {0} '{1}'.".
70                  format(table.get("type", ""), table.get("title", "")))
71     data = input_data.filter_data(table)
72
73     # Prepare the header of the tables
74     header = list()
75     for column in table["columns"]:
76         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
77
78     # Generate the data for the table according to the model in the table
79     # specification
80     job = table["data"].keys()[0]
81     build = str(table["data"][job][0])
82     try:
83         suites = input_data.suites(job, build)
84     except KeyError:
85         logging.error("    No data available. The table will not be generated.")
86         return
87
88     for suite_longname, suite in suites.iteritems():
89         # Generate data
90         suite_name = suite["name"]
91         table_lst = list()
92         for test in data[job][build].keys():
93             if data[job][build][test]["parent"] in suite_name:
94                 row_lst = list()
95                 for column in table["columns"]:
96                     try:
97                         col_data = str(data[job][build][test][column["data"].
98                                        split(" ")[1]]).replace('"', '""')
99                         if column["data"].split(" ")[1] in ("conf-history",
100                                                             "show-run"):
101                             col_data = replace(col_data, " |br| ", "",
102                                                maxreplace=1)
103                             col_data = " |prein| {0} |preout| ".\
104                                 format(col_data[:-5])
105                         row_lst.append('"{0}"'.format(col_data))
106                     except KeyError:
107                         row_lst.append("No data")
108                 table_lst.append(row_lst)
109
110         # Write the data to file
111         if table_lst:
112             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
113                                             table["output-file-ext"])
114             logging.info("      Writing file: '{}'".format(file_name))
115             with open(file_name, "w") as file_handler:
116                 file_handler.write(",".join(header) + "\n")
117                 for item in table_lst:
118                     file_handler.write(",".join(item) + "\n")
119
120     logging.info("  Done.")
121
122
123 def table_merged_details(table, input_data):
124     """Generate the table(s) with algorithm: table_merged_details
125     specified in the specification file.
126
127     :param table: Table to generate.
128     :param input_data: Data to process.
129     :type table: pandas.Series
130     :type input_data: InputData
131     """
132
133     logging.info("  Generating the table {0} ...".
134                  format(table.get("title", "")))
135
136     # Transform the data
137     logging.info("    Creating the data set for the {0} '{1}'.".
138                  format(table.get("type", ""), table.get("title", "")))
139     data = input_data.filter_data(table)
140     data = input_data.merge_data(data)
141     data.sort_index(inplace=True)
142
143     logging.info("    Creating the data set for the {0} '{1}'.".
144                  format(table.get("type", ""), table.get("title", "")))
145     suites = input_data.filter_data(table, data_set="suites")
146     suites = input_data.merge_data(suites)
147
148     # Prepare the header of the tables
149     header = list()
150     for column in table["columns"]:
151         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
152
153     for _, suite in suites.iteritems():
154         # Generate data
155         suite_name = suite["name"]
156         table_lst = list()
157         for test in data.keys():
158             if data[test]["parent"] in suite_name:
159                 row_lst = list()
160                 for column in table["columns"]:
161                     try:
162                         col_data = str(data[test][column["data"].
163                                        split(" ")[1]]).replace('"', '""')
164                         col_data = replace(col_data, "No Data",
165                                            "Not Captured     ")
166                         if column["data"].split(" ")[1] in ("conf-history",
167                                                             "show-run"):
168                             col_data = replace(col_data, " |br| ", "",
169                                                maxreplace=1)
170                             col_data = " |prein| {0} |preout| ".\
171                                 format(col_data[:-5])
172                         row_lst.append('"{0}"'.format(col_data))
173                     except KeyError:
174                         row_lst.append('"Not captured"')
175                 table_lst.append(row_lst)
176
177         # Write the data to file
178         if table_lst:
179             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
180                                             table["output-file-ext"])
181             logging.info("      Writing file: '{}'".format(file_name))
182             with open(file_name, "w") as file_handler:
183                 file_handler.write(",".join(header) + "\n")
184                 for item in table_lst:
185                     file_handler.write(",".join(item) + "\n")
186
187     logging.info("  Done.")
188
189
190 def table_performance_comparison(table, input_data):
191     """Generate the table(s) with algorithm: table_performance_comparison
192     specified in the specification file.
193
194     :param table: Table to generate.
195     :param input_data: Data to process.
196     :type table: pandas.Series
197     :type input_data: InputData
198     """
199
200     logging.info("  Generating the table {0} ...".
201                  format(table.get("title", "")))
202
203     # Transform the data
204     logging.info("    Creating the data set for the {0} '{1}'.".
205                  format(table.get("type", ""), table.get("title", "")))
206     data = input_data.filter_data(table, continue_on_error=True)
207
208     # Prepare the header of the tables
209     try:
210         header = ["Test case", ]
211
212         if table["include-tests"] == "MRR":
213             hdr_param = "Receive Rate"
214         else:
215             hdr_param = "Throughput"
216
217         history = table.get("history", None)
218         if history:
219             for item in history:
220                 header.extend(
221                     ["{0} {1} [Mpps]".format(item["title"], hdr_param),
222                      "{0} Stdev [Mpps]".format(item["title"])])
223         header.extend(
224             ["{0} {1} [Mpps]".format(table["reference"]["title"], hdr_param),
225              "{0} Stdev [Mpps]".format(table["reference"]["title"]),
226              "{0} {1} [Mpps]".format(table["compare"]["title"], hdr_param),
227              "{0} Stdev [Mpps]".format(table["compare"]["title"]),
228              "Delta [%]"])
229         header_str = ",".join(header) + "\n"
230     except (AttributeError, KeyError) as err:
231         logging.error("The model is invalid, missing parameter: {0}".
232                       format(err))
233         return
234
235     # Prepare data to the table:
236     tbl_dict = dict()
237     for job, builds in table["reference"]["data"].items():
238         for build in builds:
239             for tst_name, tst_data in data[job][str(build)].iteritems():
240                 tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\
241                     replace("-ndrpdr", "").replace("-pdrdisc", "").\
242                     replace("-ndrdisc", "").replace("-pdr", "").\
243                     replace("-ndr", "").\
244                     replace("1t1c", "1c").replace("2t1c", "1c").\
245                     replace("2t2c", "2c").replace("4t2c", "2c").\
246                     replace("4t4c", "4c").replace("8t4c", "4c")
247                 if "across topologies" in table["title"].lower():
248                     tst_name_mod = tst_name_mod.replace("2n1l-", "")
249                 if tbl_dict.get(tst_name_mod, None) is None:
250                     groups = re.search(REGEX_NIC, tst_data["parent"])
251                     nic = groups.group(0) if groups else ""
252                     name = "{0}-{1}".format(nic, "-".join(tst_data["name"].
253                                                           split("-")[:-1]))
254                     if "across testbeds" in table["title"].lower() or \
255                             "across topologies" in table["title"].lower():
256                         name = name.\
257                             replace("1t1c", "1c").replace("2t1c", "1c").\
258                             replace("2t2c", "2c").replace("4t2c", "2c").\
259                             replace("4t4c", "4c").replace("8t4c", "4c")
260                     tbl_dict[tst_name_mod] = {"name": name,
261                                               "ref-data": list(),
262                                               "cmp-data": list()}
263                 try:
264                     # TODO: Re-work when NDRPDRDISC tests are not used
265                     if table["include-tests"] == "MRR":
266                         tbl_dict[tst_name_mod]["ref-data"]. \
267                             append(tst_data["result"]["receive-rate"].avg)
268                     elif table["include-tests"] == "PDR":
269                         if tst_data["type"] == "PDR":
270                             tbl_dict[tst_name_mod]["ref-data"]. \
271                                 append(tst_data["throughput"]["value"])
272                         elif tst_data["type"] == "NDRPDR":
273                             tbl_dict[tst_name_mod]["ref-data"].append(
274                                 tst_data["throughput"]["PDR"]["LOWER"])
275                     elif table["include-tests"] == "NDR":
276                         if tst_data["type"] == "NDR":
277                             tbl_dict[tst_name_mod]["ref-data"]. \
278                                 append(tst_data["throughput"]["value"])
279                         elif tst_data["type"] == "NDRPDR":
280                             tbl_dict[tst_name_mod]["ref-data"].append(
281                                 tst_data["throughput"]["NDR"]["LOWER"])
282                     else:
283                         continue
284                 except TypeError:
285                     pass  # No data in output.xml for this test
286
287     for job, builds in table["compare"]["data"].items():
288         for build in builds:
289             for tst_name, tst_data in data[job][str(build)].iteritems():
290                 tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \
291                     replace("-ndrpdr", "").replace("-pdrdisc", ""). \
292                     replace("-ndrdisc", "").replace("-pdr", ""). \
293                     replace("-ndr", "").\
294                     replace("1t1c", "1c").replace("2t1c", "1c").\
295                     replace("2t2c", "2c").replace("4t2c", "2c").\
296                     replace("4t4c", "4c").replace("8t4c", "4c")
297                 if "across topologies" in table["title"].lower():
298                     tst_name_mod = tst_name_mod.replace("2n1l-", "")
299                 try:
300                     # TODO: Re-work when NDRPDRDISC tests are not used
301                     if table["include-tests"] == "MRR":
302                         tbl_dict[tst_name_mod]["cmp-data"]. \
303                             append(tst_data["result"]["receive-rate"].avg)
304                     elif table["include-tests"] == "PDR":
305                         if tst_data["type"] == "PDR":
306                             tbl_dict[tst_name_mod]["cmp-data"]. \
307                                 append(tst_data["throughput"]["value"])
308                         elif tst_data["type"] == "NDRPDR":
309                             tbl_dict[tst_name_mod]["cmp-data"].append(
310                                 tst_data["throughput"]["PDR"]["LOWER"])
311                     elif table["include-tests"] == "NDR":
312                         if tst_data["type"] == "NDR":
313                             tbl_dict[tst_name_mod]["cmp-data"]. \
314                                 append(tst_data["throughput"]["value"])
315                         elif tst_data["type"] == "NDRPDR":
316                             tbl_dict[tst_name_mod]["cmp-data"].append(
317                                 tst_data["throughput"]["NDR"]["LOWER"])
318                     else:
319                         continue
320                 except KeyError:
321                     pass
322                 except TypeError:
323                     tbl_dict.pop(tst_name_mod, None)
324     if history:
325         for item in history:
326             for job, builds in item["data"].items():
327                 for build in builds:
328                     for tst_name, tst_data in data[job][str(build)].iteritems():
329                         tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \
330                             replace("-ndrpdr", "").replace("-pdrdisc", ""). \
331                             replace("-ndrdisc", "").replace("-pdr", ""). \
332                             replace("-ndr", "").\
333                             replace("1t1c", "1c").replace("2t1c", "1c").\
334                             replace("2t2c", "2c").replace("4t2c", "2c").\
335                             replace("4t4c", "4c").replace("8t4c", "4c")
336                         if "across topologies" in table["title"].lower():
337                             tst_name_mod = tst_name_mod.replace("2n1l-", "")
338                         if tbl_dict.get(tst_name_mod, None) is None:
339                             continue
340                         if tbl_dict[tst_name_mod].get("history", None) is None:
341                             tbl_dict[tst_name_mod]["history"] = OrderedDict()
342                         if tbl_dict[tst_name_mod]["history"].get(item["title"],
343                                                              None) is None:
344                             tbl_dict[tst_name_mod]["history"][item["title"]] = \
345                                 list()
346                         try:
347                             # TODO: Re-work when NDRPDRDISC tests are not used
348                             if table["include-tests"] == "MRR":
349                                 tbl_dict[tst_name_mod]["history"][item["title"
350                                 ]].append(tst_data["result"]["receive-rate"].
351                                           avg)
352                             elif table["include-tests"] == "PDR":
353                                 if tst_data["type"] == "PDR":
354                                     tbl_dict[tst_name_mod]["history"][
355                                         item["title"]].\
356                                         append(tst_data["throughput"]["value"])
357                                 elif tst_data["type"] == "NDRPDR":
358                                     tbl_dict[tst_name_mod]["history"][item[
359                                         "title"]].append(tst_data["throughput"][
360                                         "PDR"]["LOWER"])
361                             elif table["include-tests"] == "NDR":
362                                 if tst_data["type"] == "NDR":
363                                     tbl_dict[tst_name_mod]["history"][
364                                         item["title"]].\
365                                         append(tst_data["throughput"]["value"])
366                                 elif tst_data["type"] == "NDRPDR":
367                                     tbl_dict[tst_name_mod]["history"][item[
368                                         "title"]].append(tst_data["throughput"][
369                                         "NDR"]["LOWER"])
370                             else:
371                                 continue
372                         except (TypeError, KeyError):
373                             pass
374
375     tbl_lst = list()
376     for tst_name in tbl_dict.keys():
377         item = [tbl_dict[tst_name]["name"], ]
378         if history:
379             if tbl_dict[tst_name].get("history", None) is not None:
380                 for hist_data in tbl_dict[tst_name]["history"].values():
381                     if hist_data:
382                         item.append(round(mean(hist_data) / 1000000, 2))
383                         item.append(round(stdev(hist_data) / 1000000, 2))
384                     else:
385                         item.extend([None, None])
386             else:
387                 item.extend([None, None])
388         data_t = tbl_dict[tst_name]["ref-data"]
389         if data_t:
390             item.append(round(mean(data_t) / 1000000, 2))
391             item.append(round(stdev(data_t) / 1000000, 2))
392         else:
393             item.extend([None, None])
394         data_t = tbl_dict[tst_name]["cmp-data"]
395         if data_t:
396             item.append(round(mean(data_t) / 1000000, 2))
397             item.append(round(stdev(data_t) / 1000000, 2))
398         else:
399             item.extend([None, None])
400         if item[-4] is not None and item[-2] is not None and item[-4] != 0:
401             item.append(int(relative_change(float(item[-4]), float(item[-2]))))
402         if len(item) == len(header):
403             tbl_lst.append(item)
404
405     # Sort the table according to the relative change
406     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
407
408     # Generate csv tables:
409     csv_file = "{0}.csv".format(table["output-file"])
410     with open(csv_file, "w") as file_handler:
411         file_handler.write(header_str)
412         for test in tbl_lst:
413             file_handler.write(",".join([str(item) for item in test]) + "\n")
414
415     convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
416
417
418 def table_nics_comparison(table, input_data):
419     """Generate the table(s) with algorithm: table_nics_comparison
420     specified in the specification file.
421
422     :param table: Table to generate.
423     :param input_data: Data to process.
424     :type table: pandas.Series
425     :type input_data: InputData
426     """
427
428     logging.info("  Generating the table {0} ...".
429                  format(table.get("title", "")))
430
431     # Transform the data
432     logging.info("    Creating the data set for the {0} '{1}'.".
433                  format(table.get("type", ""), table.get("title", "")))
434     data = input_data.filter_data(table, continue_on_error=True)
435
436     # Prepare the header of the tables
437     try:
438         header = ["Test case", ]
439
440         if table["include-tests"] == "MRR":
441             hdr_param = "Receive Rate"
442         else:
443             hdr_param = "Throughput"
444
445         header.extend(
446             ["{0} {1} [Mpps]".format(table["reference"]["title"], hdr_param),
447              "{0} Stdev [Mpps]".format(table["reference"]["title"]),
448              "{0} {1} [Mpps]".format(table["compare"]["title"], hdr_param),
449              "{0} Stdev [Mpps]".format(table["compare"]["title"]),
450              "Delta [%]"])
451         header_str = ",".join(header) + "\n"
452     except (AttributeError, KeyError) as err:
453         logging.error("The model is invalid, missing parameter: {0}".
454                       format(err))
455         return
456
457     # Prepare data to the table:
458     tbl_dict = dict()
459     for job, builds in table["data"].items():
460         for build in builds:
461             for tst_name, tst_data in data[job][str(build)].iteritems():
462                 tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\
463                     replace("-ndrpdr", "").replace("-pdrdisc", "").\
464                     replace("-ndrdisc", "").replace("-pdr", "").\
465                     replace("-ndr", "").\
466                     replace("1t1c", "1c").replace("2t1c", "1c").\
467                     replace("2t2c", "2c").replace("4t2c", "2c").\
468                     replace("4t4c", "4c").replace("8t4c", "4c")
469                 tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod)
470                 if tbl_dict.get(tst_name_mod, None) is None:
471                     name = "-".join(tst_data["name"].split("-")[:-1])
472                     tbl_dict[tst_name_mod] = {"name": name,
473                                               "ref-data": list(),
474                                               "cmp-data": list()}
475                 try:
476                     if table["include-tests"] == "MRR":
477                         result = tst_data["result"]["receive-rate"].avg
478                     elif table["include-tests"] == "PDR":
479                         result = tst_data["throughput"]["PDR"]["LOWER"]
480                     elif table["include-tests"] == "NDR":
481                         result = tst_data["throughput"]["NDR"]["LOWER"]
482                     else:
483                         result = None
484
485                     if result:
486                         if table["reference"]["nic"] in tst_data["tags"]:
487                             tbl_dict[tst_name_mod]["ref-data"].append(result)
488                         elif table["compare"]["nic"] in tst_data["tags"]:
489                             tbl_dict[tst_name_mod]["cmp-data"].append(result)
490                 except (TypeError, KeyError) as err:
491                     logging.debug("No data for {0}".format(tst_name))
492                     logging.debug(repr(err))
493                     # No data in output.xml for this test
494
495     tbl_lst = list()
496     for tst_name in tbl_dict.keys():
497         item = [tbl_dict[tst_name]["name"], ]
498         data_t = tbl_dict[tst_name]["ref-data"]
499         if data_t:
500             item.append(round(mean(data_t) / 1000000, 2))
501             item.append(round(stdev(data_t) / 1000000, 2))
502         else:
503             item.extend([None, None])
504         data_t = tbl_dict[tst_name]["cmp-data"]
505         if data_t:
506             item.append(round(mean(data_t) / 1000000, 2))
507             item.append(round(stdev(data_t) / 1000000, 2))
508         else:
509             item.extend([None, None])
510         if item[-4] is not None and item[-2] is not None and item[-4] != 0:
511             item.append(int(relative_change(float(item[-4]), float(item[-2]))))
512         if len(item) == len(header):
513             tbl_lst.append(item)
514
515     # Sort the table according to the relative change
516     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
517
518     # Generate csv tables:
519     csv_file = "{0}.csv".format(table["output-file"])
520     with open(csv_file, "w") as file_handler:
521         file_handler.write(header_str)
522         for test in tbl_lst:
523             file_handler.write(",".join([str(item) for item in test]) + "\n")
524
525     convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
526
527
528 def table_soak_vs_ndr(table, input_data):
529     """Generate the table(s) with algorithm: table_soak_vs_ndr
530     specified in the specification file.
531
532     :param table: Table to generate.
533     :param input_data: Data to process.
534     :type table: pandas.Series
535     :type input_data: InputData
536     """
537
538     logging.info("  Generating the table {0} ...".
539                  format(table.get("title", "")))
540
541     # Transform the data
542     logging.info("    Creating the data set for the {0} '{1}'.".
543                  format(table.get("type", ""), table.get("title", "")))
544     data = input_data.filter_data(table, continue_on_error=True)
545
546     # Prepare the header of the table
547     try:
548         header = [
549             "Test case",
550             "{0} Throughput [Mpps]".format(table["reference"]["title"]),
551             "{0} Stdev [Mpps]".format(table["reference"]["title"]),
552             "{0} Throughput [Mpps]".format(table["compare"]["title"]),
553             "{0} Stdev [Mpps]".format(table["compare"]["title"]),
554             "Delta [%]"]
555         header_str = ",".join(header) + "\n"
556     except (AttributeError, KeyError) as err:
557         logging.error("The model is invalid, missing parameter: {0}".
558                       format(err))
559         return
560
561     # Create a list of available SOAK test results:
562     tbl_dict = dict()
563     for job, builds in table["compare"]["data"].items():
564         for build in builds:
565             for tst_name, tst_data in data[job][str(build)].iteritems():
566                 if tst_data["type"] == "SOAK":
567                     tst_name_mod = tst_name.replace("-soak", "")
568                     if tbl_dict.get(tst_name_mod, None) is None:
569                         groups = re.search(REGEX_NIC, tst_data["parent"])
570                         nic = groups.group(0) if groups else ""
571                         name = "{0}-{1}".format(nic, "-".join(tst_data["name"].
572                                                               split("-")[:-1]))
573                         tbl_dict[tst_name_mod] = {
574                             "name": name,
575                             "ref-data": list(),
576                             "cmp-data": list()
577                         }
578                     try:
579                         tbl_dict[tst_name_mod]["cmp-data"].append(
580                             tst_data["throughput"]["LOWER"])
581                     except (KeyError, TypeError):
582                         pass
583     tests_lst = tbl_dict.keys()
584
585     # Add corresponding NDR test results:
586     for job, builds in table["reference"]["data"].items():
587         for build in builds:
588             for tst_name, tst_data in data[job][str(build)].iteritems():
589                 tst_name_mod = tst_name.replace("-ndrpdr", "").\
590                     replace("-mrr", "")
591                 if tst_name_mod in tests_lst:
592                     try:
593                         if tst_data["type"] in ("NDRPDR", "MRR", "BMRR"):
594                             if table["include-tests"] == "MRR":
595                                 result = tst_data["result"]["receive-rate"].avg
596                             elif table["include-tests"] == "PDR":
597                                 result = tst_data["throughput"]["PDR"]["LOWER"]
598                             elif table["include-tests"] == "NDR":
599                                 result = tst_data["throughput"]["NDR"]["LOWER"]
600                             else:
601                                 result = None
602                             if result is not None:
603                                 tbl_dict[tst_name_mod]["ref-data"].append(
604                                     result)
605                     except (KeyError, TypeError):
606                         continue
607
608     tbl_lst = list()
609     for tst_name in tbl_dict.keys():
610         item = [tbl_dict[tst_name]["name"], ]
611         data_r = tbl_dict[tst_name]["ref-data"]
612         if data_r:
613             data_r_mean = mean(data_r)
614             item.append(round(data_r_mean / 1000000, 2))
615             item.append(round(stdev(data_r) / 1000000, 2))
616         else:
617             data_r_mean = None
618             item.extend([None, None])
619         data_c = tbl_dict[tst_name]["cmp-data"]
620         if data_c:
621             data_c_mean = mean(data_c)
622             item.append(round(data_c_mean / 1000000, 2))
623             item.append(round(stdev(data_c) / 1000000, 2))
624         else:
625             data_c_mean = None
626             item.extend([None, None])
627         if data_r_mean and data_c_mean is not None:
628             item.append(round(relative_change(data_r_mean, data_c_mean), 2))
629             tbl_lst.append(item)
630
631     # Sort the table according to the relative change
632     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
633
634     # Generate csv tables:
635     csv_file = "{0}.csv".format(table["output-file"])
636     with open(csv_file, "w") as file_handler:
637         file_handler.write(header_str)
638         for test in tbl_lst:
639             file_handler.write(",".join([str(item) for item in test]) + "\n")
640
641     convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
642
643
644 def table_performance_trending_dashboard(table, input_data):
645     """Generate the table(s) with algorithm:
646     table_performance_trending_dashboard
647     specified in the specification file.
648
649     :param table: Table to generate.
650     :param input_data: Data to process.
651     :type table: pandas.Series
652     :type input_data: InputData
653     """
654
655     logging.info("  Generating the table {0} ...".
656                  format(table.get("title", "")))
657
658     # Transform the data
659     logging.info("    Creating the data set for the {0} '{1}'.".
660                  format(table.get("type", ""), table.get("title", "")))
661     data = input_data.filter_data(table, continue_on_error=True)
662
663     # Prepare the header of the tables
664     header = ["Test Case",
665               "Trend [Mpps]",
666               "Short-Term Change [%]",
667               "Long-Term Change [%]",
668               "Regressions [#]",
669               "Progressions [#]"
670               ]
671     header_str = ",".join(header) + "\n"
672
673     # Prepare data to the table:
674     tbl_dict = dict()
675     for job, builds in table["data"].items():
676         for build in builds:
677             for tst_name, tst_data in data[job][str(build)].iteritems():
678                 if tst_name.lower() in table["ignore-list"]:
679                     continue
680                 if tbl_dict.get(tst_name, None) is None:
681                     groups = re.search(REGEX_NIC, tst_data["parent"])
682                     if not groups:
683                         continue
684                     nic = groups.group(0)
685                     tbl_dict[tst_name] = {
686                         "name": "{0}-{1}".format(nic, tst_data["name"]),
687                         "data": OrderedDict()}
688                 try:
689                     tbl_dict[tst_name]["data"][str(build)] = \
690                         tst_data["result"]["receive-rate"]
691                 except (TypeError, KeyError):
692                     pass  # No data in output.xml for this test
693
694     tbl_lst = list()
695     for tst_name in tbl_dict.keys():
696         data_t = tbl_dict[tst_name]["data"]
697         if len(data_t) < 2:
698             continue
699
700         classification_lst, avgs = classify_anomalies(data_t)
701
702         win_size = min(len(data_t), table["window"])
703         long_win_size = min(len(data_t), table["long-trend-window"])
704
705         try:
706             max_long_avg = max(
707                 [x for x in avgs[-long_win_size:-win_size]
708                  if not isnan(x)])
709         except ValueError:
710             max_long_avg = nan
711         last_avg = avgs[-1]
712         avg_week_ago = avgs[max(-win_size, -len(avgs))]
713
714         if isnan(last_avg) or isnan(avg_week_ago) or avg_week_ago == 0.0:
715             rel_change_last = nan
716         else:
717             rel_change_last = round(
718                 ((last_avg - avg_week_ago) / avg_week_ago) * 100, 2)
719
720         if isnan(max_long_avg) or isnan(last_avg) or max_long_avg == 0.0:
721             rel_change_long = nan
722         else:
723             rel_change_long = round(
724                 ((last_avg - max_long_avg) / max_long_avg) * 100, 2)
725
726         if classification_lst:
727             if isnan(rel_change_last) and isnan(rel_change_long):
728                 continue
729             if (isnan(last_avg) or
730                 isnan(rel_change_last) or
731                 isnan(rel_change_long)):
732                 continue
733             tbl_lst.append(
734                 [tbl_dict[tst_name]["name"],
735                  round(last_avg / 1000000, 2),
736                  rel_change_last,
737                  rel_change_long,
738                  classification_lst[-win_size:].count("regression"),
739                  classification_lst[-win_size:].count("progression")])
740
741     tbl_lst.sort(key=lambda rel: rel[0])
742
743     tbl_sorted = list()
744     for nrr in range(table["window"], -1, -1):
745         tbl_reg = [item for item in tbl_lst if item[4] == nrr]
746         for nrp in range(table["window"], -1, -1):
747             tbl_out = [item for item in tbl_reg if item[5] == nrp]
748             tbl_out.sort(key=lambda rel: rel[2])
749             tbl_sorted.extend(tbl_out)
750
751     file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
752
753     logging.info("    Writing file: '{0}'".format(file_name))
754     with open(file_name, "w") as file_handler:
755         file_handler.write(header_str)
756         for test in tbl_sorted:
757             file_handler.write(",".join([str(item) for item in test]) + '\n')
758
759     txt_file_name = "{0}.txt".format(table["output-file"])
760     logging.info("    Writing file: '{0}'".format(txt_file_name))
761     convert_csv_to_pretty_txt(file_name, txt_file_name)
762
763
764 def _generate_url(base, testbed, test_name):
765     """Generate URL to a trending plot from the name of the test case.
766
767     :param base: The base part of URL common to all test cases.
768     :param testbed: The testbed used for testing.
769     :param test_name: The name of the test case.
770     :type base: str
771     :type testbed: str
772     :type test_name: str
773     :returns: The URL to the plot with the trending data for the given test
774         case.
775     :rtype str
776     """
777
778     url = base
779     file_name = ""
780     anchor = ".html#"
781     feature = ""
782
783     if "lbdpdk" in test_name or "lbvpp" in test_name:
784         file_name = "link_bonding"
785
786     elif "114b" in test_name and "vhost" in test_name:
787         file_name = "vts"
788
789     elif "testpmd" in test_name or "l3fwd" in test_name:
790         file_name = "dpdk"
791
792     elif "memif" in test_name:
793         file_name = "container_memif"
794         feature = "-base"
795
796     elif "srv6" in test_name:
797         file_name = "srv6"
798
799     elif "vhost" in test_name:
800         if "l2xcbase" in test_name or "l2bdbasemaclrn" in test_name:
801             file_name = "vm_vhost_l2"
802             if "114b" in test_name:
803                 feature = ""
804             elif "l2xcbase" in test_name and "x520" in test_name:
805                 feature = "-base-l2xc"
806             elif "l2bdbasemaclrn" in test_name and "x520" in test_name:
807                 feature = "-base-l2bd"
808             else:
809                 feature = "-base"
810         elif "ip4base" in test_name:
811             file_name = "vm_vhost_ip4"
812             feature = "-base"
813
814     elif "ipsec" in test_name:
815         file_name = "ipsec"
816         feature = "-base-scale"
817
818     elif "ethip4lispip" in test_name or "ethip4vxlan" in test_name:
819         file_name = "ip4_tunnels"
820         feature = "-base"
821
822     elif "ip4base" in test_name or "ip4scale" in test_name:
823         file_name = "ip4"
824         if "xl710" in test_name:
825             feature = "-base-scale-features"
826         elif "iacl" in test_name:
827             feature = "-features-iacl"
828         elif "oacl" in test_name:
829             feature = "-features-oacl"
830         elif "snat" in test_name or "cop" in test_name:
831             feature = "-features"
832         else:
833             feature = "-base-scale"
834
835     elif "ip6base" in test_name or "ip6scale" in test_name:
836         file_name = "ip6"
837         feature = "-base-scale"
838
839     elif "l2xcbase" in test_name or "l2xcscale" in test_name \
840             or "l2bdbasemaclrn" in test_name or "l2bdscale" in test_name \
841             or "l2dbbasemaclrn" in test_name or "l2dbscale" in test_name:
842         file_name = "l2"
843         if "macip" in test_name:
844             feature = "-features-macip"
845         elif "iacl" in test_name:
846             feature = "-features-iacl"
847         elif "oacl" in test_name:
848             feature = "-features-oacl"
849         else:
850             feature = "-base-scale"
851
852     if "x520" in test_name:
853         nic = "x520-"
854     elif "x710" in test_name:
855         nic = "x710-"
856     elif "xl710" in test_name:
857         nic = "xl710-"
858     elif "xxv710" in test_name:
859         nic = "xxv710-"
860     elif "vic1227" in test_name:
861         nic = "vic1227-"
862     elif "vic1385" in test_name:
863         nic = "vic1385-"
864     else:
865         nic = ""
866     anchor += nic
867
868     if "64b" in test_name:
869         framesize = "64b"
870     elif "78b" in test_name:
871         framesize = "78b"
872     elif "imix" in test_name:
873         framesize = "imix"
874     elif "9000b" in test_name:
875         framesize = "9000b"
876     elif "1518b" in test_name:
877         framesize = "1518b"
878     elif "114b" in test_name:
879         framesize = "114b"
880     else:
881         framesize = ""
882     anchor += framesize + '-'
883
884     if "1t1c" in test_name:
885         anchor += "1t1c"
886     elif "2t2c" in test_name:
887         anchor += "2t2c"
888     elif "4t4c" in test_name:
889         anchor += "4t4c"
890     elif "2t1c" in test_name:
891         anchor += "2t1c"
892     elif "4t2c" in test_name:
893         anchor += "4t2c"
894     elif "8t4c" in test_name:
895         anchor += "8t4c"
896
897     return url + file_name + '-' + testbed + '-' + nic + framesize + feature + \
898            anchor + feature
899
900
901 def table_performance_trending_dashboard_html(table, input_data):
902     """Generate the table(s) with algorithm:
903     table_performance_trending_dashboard_html specified in the specification
904     file.
905
906     :param table: Table to generate.
907     :param input_data: Data to process.
908     :type table: dict
909     :type input_data: InputData
910     """
911
912     testbed = table.get("testbed", None)
913     if testbed is None:
914         logging.error("The testbed is not defined for the table '{0}'.".
915                       format(table.get("title", "")))
916         return
917
918     logging.info("  Generating the table {0} ...".
919                  format(table.get("title", "")))
920
921     try:
922         with open(table["input-file"], 'rb') as csv_file:
923             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
924             csv_lst = [item for item in csv_content]
925     except KeyError:
926         logging.warning("The input file is not defined.")
927         return
928     except csv.Error as err:
929         logging.warning("Not possible to process the file '{0}'.\n{1}".
930                         format(table["input-file"], err))
931         return
932
933     # Table:
934     dashboard = ET.Element("table", attrib=dict(width="100%", border='0'))
935
936     # Table header:
937     tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor="#7eade7"))
938     for idx, item in enumerate(csv_lst[0]):
939         alignment = "left" if idx == 0 else "center"
940         th = ET.SubElement(tr, "th", attrib=dict(align=alignment))
941         th.text = item
942
943     # Rows:
944     colors = {"regression": ("#ffcccc", "#ff9999"),
945               "progression": ("#c6ecc6", "#9fdf9f"),
946               "normal": ("#e9f1fb", "#d4e4f7")}
947     for r_idx, row in enumerate(csv_lst[1:]):
948         if int(row[4]):
949             color = "regression"
950         elif int(row[5]):
951             color = "progression"
952         else:
953             color = "normal"
954         background = colors[color][r_idx % 2]
955         tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor=background))
956
957         # Columns:
958         for c_idx, item in enumerate(row):
959             alignment = "left" if c_idx == 0 else "center"
960             td = ET.SubElement(tr, "td", attrib=dict(align=alignment))
961             # Name:
962             if c_idx == 0:
963                 url = _generate_url("../trending/", testbed, item)
964                 ref = ET.SubElement(td, "a", attrib=dict(href=url))
965                 ref.text = item
966             else:
967                 td.text = item
968     try:
969         with open(table["output-file"], 'w') as html_file:
970             logging.info("    Writing file: '{0}'".format(table["output-file"]))
971             html_file.write(".. raw:: html\n\n\t")
972             html_file.write(ET.tostring(dashboard))
973             html_file.write("\n\t<p><br><br></p>\n")
974     except KeyError:
975         logging.warning("The output file is not defined.")
976         return
977
978
979 def table_last_failed_tests(table, input_data):
980     """Generate the table(s) with algorithm: table_last_failed_tests
981     specified in the specification file.
982
983     :param table: Table to generate.
984     :param input_data: Data to process.
985     :type table: pandas.Series
986     :type input_data: InputData
987     """
988
989     logging.info("  Generating the table {0} ...".
990                  format(table.get("title", "")))
991
992     # Transform the data
993     logging.info("    Creating the data set for the {0} '{1}'.".
994                  format(table.get("type", ""), table.get("title", "")))
995     data = input_data.filter_data(table, continue_on_error=True)
996
997     if data is None or data.empty:
998         logging.warn("    No data for the {0} '{1}'.".
999                      format(table.get("type", ""), table.get("title", "")))
1000         return
1001
1002     tbl_list = list()
1003     for job, builds in table["data"].items():
1004         for build in builds:
1005             build = str(build)
1006             try:
1007                 version = input_data.metadata(job, build).get("version", "")
1008             except KeyError:
1009                 logging.error("Data for {job}: {build} is not present.".
1010                               format(job=job, build=build))
1011                 return
1012             tbl_list.append(build)
1013             tbl_list.append(version)
1014             for tst_name, tst_data in data[job][build].iteritems():
1015                 if tst_data["status"] != "FAIL":
1016                     continue
1017                 groups = re.search(REGEX_NIC, tst_data["parent"])
1018                 if not groups:
1019                     continue
1020                 nic = groups.group(0)
1021                 tbl_list.append("{0}-{1}".format(nic, tst_data["name"]))
1022
1023     file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
1024     logging.info("    Writing file: '{0}'".format(file_name))
1025     with open(file_name, "w") as file_handler:
1026         for test in tbl_list:
1027             file_handler.write(test + '\n')
1028
1029
1030 def table_failed_tests(table, input_data):
1031     """Generate the table(s) with algorithm: table_failed_tests
1032     specified in the specification file.
1033
1034     :param table: Table to generate.
1035     :param input_data: Data to process.
1036     :type table: pandas.Series
1037     :type input_data: InputData
1038     """
1039
1040     logging.info("  Generating the table {0} ...".
1041                  format(table.get("title", "")))
1042
1043     # Transform the data
1044     logging.info("    Creating the data set for the {0} '{1}'.".
1045                  format(table.get("type", ""), table.get("title", "")))
1046     data = input_data.filter_data(table, continue_on_error=True)
1047
1048     # Prepare the header of the tables
1049     header = ["Test Case",
1050               "Failures [#]",
1051               "Last Failure [Time]",
1052               "Last Failure [VPP-Build-Id]",
1053               "Last Failure [CSIT-Job-Build-Id]"]
1054
1055     # Generate the data for the table according to the model in the table
1056     # specification
1057
1058     now = dt.utcnow()
1059     timeperiod = timedelta(int(table.get("window", 7)))
1060
1061     tbl_dict = dict()
1062     for job, builds in table["data"].items():
1063         for build in builds:
1064             build = str(build)
1065             for tst_name, tst_data in data[job][build].iteritems():
1066                 if tst_name.lower() in table["ignore-list"]:
1067                     continue
1068                 if tbl_dict.get(tst_name, None) is None:
1069                     groups = re.search(REGEX_NIC, tst_data["parent"])
1070                     if not groups:
1071                         continue
1072                     nic = groups.group(0)
1073                     tbl_dict[tst_name] = {
1074                         "name": "{0}-{1}".format(nic, tst_data["name"]),
1075                         "data": OrderedDict()}
1076                 try:
1077                     generated = input_data.metadata(job, build).\
1078                         get("generated", "")
1079                     if not generated:
1080                         continue
1081                     then = dt.strptime(generated, "%Y%m%d %H:%M")
1082                     if (now - then) <= timeperiod:
1083                         tbl_dict[tst_name]["data"][build] = (
1084                             tst_data["status"],
1085                             generated,
1086                             input_data.metadata(job, build).get("version", ""),
1087                             build)
1088                 except (TypeError, KeyError) as err:
1089                     logging.warning("tst_name: {} - err: {}".
1090                                     format(tst_name, repr(err)))
1091
1092     max_fails = 0
1093     tbl_lst = list()
1094     for tst_data in tbl_dict.values():
1095         fails_nr = 0
1096         for val in tst_data["data"].values():
1097             if val[0] == "FAIL":
1098                 fails_nr += 1
1099                 fails_last_date = val[1]
1100                 fails_last_vpp = val[2]
1101                 fails_last_csit = val[3]
1102         if fails_nr:
1103             max_fails = fails_nr if fails_nr > max_fails else max_fails
1104             tbl_lst.append([tst_data["name"],
1105                             fails_nr,
1106                             fails_last_date,
1107                             fails_last_vpp,
1108                             "mrr-daily-build-{0}".format(fails_last_csit)])
1109
1110     tbl_lst.sort(key=lambda rel: rel[2], reverse=True)
1111     tbl_sorted = list()
1112     for nrf in range(max_fails, -1, -1):
1113         tbl_fails = [item for item in tbl_lst if item[1] == nrf]
1114         tbl_sorted.extend(tbl_fails)
1115     file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
1116
1117     logging.info("    Writing file: '{0}'".format(file_name))
1118     with open(file_name, "w") as file_handler:
1119         file_handler.write(",".join(header) + "\n")
1120         for test in tbl_sorted:
1121             file_handler.write(",".join([str(item) for item in test]) + '\n')
1122
1123     txt_file_name = "{0}.txt".format(table["output-file"])
1124     logging.info("    Writing file: '{0}'".format(txt_file_name))
1125     convert_csv_to_pretty_txt(file_name, txt_file_name)
1126
1127
1128 def table_failed_tests_html(table, input_data):
1129     """Generate the table(s) with algorithm: table_failed_tests_html
1130     specified in the specification file.
1131
1132     :param table: Table to generate.
1133     :param input_data: Data to process.
1134     :type table: pandas.Series
1135     :type input_data: InputData
1136     """
1137
1138     testbed = table.get("testbed", None)
1139     if testbed is None:
1140         logging.error("The testbed is not defined for the table '{0}'.".
1141                       format(table.get("title", "")))
1142         return
1143
1144     logging.info("  Generating the table {0} ...".
1145                  format(table.get("title", "")))
1146
1147     try:
1148         with open(table["input-file"], 'rb') as csv_file:
1149             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
1150             csv_lst = [item for item in csv_content]
1151     except KeyError:
1152         logging.warning("The input file is not defined.")
1153         return
1154     except csv.Error as err:
1155         logging.warning("Not possible to process the file '{0}'.\n{1}".
1156                         format(table["input-file"], err))
1157         return
1158
1159     # Table:
1160     failed_tests = ET.Element("table", attrib=dict(width="100%", border='0'))
1161
1162     # Table header:
1163     tr = ET.SubElement(failed_tests, "tr", attrib=dict(bgcolor="#7eade7"))
1164     for idx, item in enumerate(csv_lst[0]):
1165         alignment = "left" if idx == 0 else "center"
1166         th = ET.SubElement(tr, "th", attrib=dict(align=alignment))
1167         th.text = item
1168
1169     # Rows:
1170     colors = ("#e9f1fb", "#d4e4f7")
1171     for r_idx, row in enumerate(csv_lst[1:]):
1172         background = colors[r_idx % 2]
1173         tr = ET.SubElement(failed_tests, "tr", attrib=dict(bgcolor=background))
1174
1175         # Columns:
1176         for c_idx, item in enumerate(row):
1177             alignment = "left" if c_idx == 0 else "center"
1178             td = ET.SubElement(tr, "td", attrib=dict(align=alignment))
1179             # Name:
1180             if c_idx == 0:
1181                 url = _generate_url("../trending/", testbed, item)
1182                 ref = ET.SubElement(td, "a", attrib=dict(href=url))
1183                 ref.text = item
1184             else:
1185                 td.text = item
1186     try:
1187         with open(table["output-file"], 'w') as html_file:
1188             logging.info("    Writing file: '{0}'".format(table["output-file"]))
1189             html_file.write(".. raw:: html\n\n\t")
1190             html_file.write(ET.tostring(failed_tests))
1191             html_file.write("\n\t<p><br><br></p>\n")
1192     except KeyError:
1193         logging.warning("The output file is not defined.")
1194         return

©2016 FD.io a Linux Foundation Collaborative Project. All Rights Reserved.
Linux Foundation is a registered trademark of The Linux Foundation. Linux is a registered trademark of Linus Torvalds.
Please see our privacy policy and terms of use.