7fc1d0cfd713701ed811cceb9d80f23670571670
[csit.git] / resources / tools / presentation / generator_tables.py
1 # Copyright (c) 2019 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
5 #
6 #     http://www.apache.org/licenses/LICENSE-2.0
7 #
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
13
14 """Algorithms to generate tables.
15 """
16
17
18 import logging
19 import csv
20 import re
21
22 from string import replace
23 from collections import OrderedDict
24 from numpy import nan, isnan
25 from xml.etree import ElementTree as ET
26 from datetime import datetime as dt
27 from datetime import timedelta
28
29 from utils import mean, stdev, relative_change, classify_anomalies, \
30     convert_csv_to_pretty_txt
31
32
33 REGEX_NIC = re.compile(r'\d*ge\dp\d\D*\d*')
34
35
36 def generate_tables(spec, data):
37     """Generate all tables specified in the specification file.
38
39     :param spec: Specification read from the specification file.
40     :param data: Data to process.
41     :type spec: Specification
42     :type data: InputData
43     """
44
45     logging.info("Generating the tables ...")
46     for table in spec.tables:
47         try:
48             eval(table["algorithm"])(table, data)
49         except NameError as err:
50             logging.error("Probably algorithm '{alg}' is not defined: {err}".
51                           format(alg=table["algorithm"], err=repr(err)))
52     logging.info("Done.")
53
54
55 def table_details(table, input_data):
56     """Generate the table(s) with algorithm: table_detailed_test_results
57     specified in the specification file.
58
59     :param table: Table to generate.
60     :param input_data: Data to process.
61     :type table: pandas.Series
62     :type input_data: InputData
63     """
64
65     logging.info("  Generating the table {0} ...".
66                  format(table.get("title", "")))
67
68     # Transform the data
69     logging.info("    Creating the data set for the {0} '{1}'.".
70                  format(table.get("type", ""), table.get("title", "")))
71     data = input_data.filter_data(table)
72
73     # Prepare the header of the tables
74     header = list()
75     for column in table["columns"]:
76         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
77
78     # Generate the data for the table according to the model in the table
79     # specification
80     job = table["data"].keys()[0]
81     build = str(table["data"][job][0])
82     try:
83         suites = input_data.suites(job, build)
84     except KeyError:
85         logging.error("    No data available. The table will not be generated.")
86         return
87
88     for suite_longname, suite in suites.iteritems():
89         # Generate data
90         suite_name = suite["name"]
91         table_lst = list()
92         for test in data[job][build].keys():
93             if data[job][build][test]["parent"] in suite_name:
94                 row_lst = list()
95                 for column in table["columns"]:
96                     try:
97                         col_data = str(data[job][build][test][column["data"].
98                                        split(" ")[1]]).replace('"', '""')
99                         if column["data"].split(" ")[1] in ("vat-history",
100                                                             "show-run"):
101                             col_data = replace(col_data, " |br| ", "",
102                                                maxreplace=1)
103                             col_data = " |prein| {0} |preout| ".\
104                                 format(col_data[:-5])
105                         row_lst.append('"{0}"'.format(col_data))
106                     except KeyError:
107                         row_lst.append("No data")
108                 table_lst.append(row_lst)
109
110         # Write the data to file
111         if table_lst:
112             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
113                                             table["output-file-ext"])
114             logging.info("      Writing file: '{}'".format(file_name))
115             with open(file_name, "w") as file_handler:
116                 file_handler.write(",".join(header) + "\n")
117                 for item in table_lst:
118                     file_handler.write(",".join(item) + "\n")
119
120     logging.info("  Done.")
121
122
123 def table_merged_details(table, input_data):
124     """Generate the table(s) with algorithm: table_merged_details
125     specified in the specification file.
126
127     :param table: Table to generate.
128     :param input_data: Data to process.
129     :type table: pandas.Series
130     :type input_data: InputData
131     """
132
133     logging.info("  Generating the table {0} ...".
134                  format(table.get("title", "")))
135
136     # Transform the data
137     logging.info("    Creating the data set for the {0} '{1}'.".
138                  format(table.get("type", ""), table.get("title", "")))
139     data = input_data.filter_data(table)
140     data = input_data.merge_data(data)
141     data.sort_index(inplace=True)
142
143     logging.info("    Creating the data set for the {0} '{1}'.".
144                  format(table.get("type", ""), table.get("title", "")))
145     suites = input_data.filter_data(table, data_set="suites")
146     suites = input_data.merge_data(suites)
147
148     # Prepare the header of the tables
149     header = list()
150     for column in table["columns"]:
151         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
152
153     for _, suite in suites.iteritems():
154         # Generate data
155         suite_name = suite["name"]
156         table_lst = list()
157         for test in data.keys():
158             if data[test]["parent"] in suite_name:
159                 row_lst = list()
160                 for column in table["columns"]:
161                     try:
162                         col_data = str(data[test][column["data"].
163                                        split(" ")[1]]).replace('"', '""')
164                         if column["data"].split(" ")[1] in ("vat-history",
165                                                             "show-run"):
166                             col_data = replace(col_data, " |br| ", "",
167                                                maxreplace=1)
168                             col_data = " |prein| {0} |preout| ".\
169                                 format(col_data[:-5])
170                         row_lst.append('"{0}"'.format(col_data))
171                     except KeyError:
172                         row_lst.append("No data")
173                 table_lst.append(row_lst)
174
175         # Write the data to file
176         if table_lst:
177             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
178                                             table["output-file-ext"])
179             logging.info("      Writing file: '{}'".format(file_name))
180             with open(file_name, "w") as file_handler:
181                 file_handler.write(",".join(header) + "\n")
182                 for item in table_lst:
183                     file_handler.write(",".join(item) + "\n")
184
185     logging.info("  Done.")
186
187
188 def table_performance_comparison(table, input_data):
189     """Generate the table(s) with algorithm: table_performance_comparison
190     specified in the specification file.
191
192     :param table: Table to generate.
193     :param input_data: Data to process.
194     :type table: pandas.Series
195     :type input_data: InputData
196     """
197
198     logging.info("  Generating the table {0} ...".
199                  format(table.get("title", "")))
200
201     # Transform the data
202     logging.info("    Creating the data set for the {0} '{1}'.".
203                  format(table.get("type", ""), table.get("title", "")))
204     data = input_data.filter_data(table, continue_on_error=True)
205
206     # Prepare the header of the tables
207     try:
208         header = ["Test case", ]
209
210         if table["include-tests"] == "MRR":
211             hdr_param = "Receive Rate"
212         else:
213             hdr_param = "Throughput"
214
215         history = table.get("history", None)
216         if history:
217             for item in history:
218                 header.extend(
219                     ["{0} {1} [Mpps]".format(item["title"], hdr_param),
220                      "{0} Stdev [Mpps]".format(item["title"])])
221         header.extend(
222             ["{0} {1} [Mpps]".format(table["reference"]["title"], hdr_param),
223              "{0} Stdev [Mpps]".format(table["reference"]["title"]),
224              "{0} {1} [Mpps]".format(table["compare"]["title"], hdr_param),
225              "{0} Stdev [Mpps]".format(table["compare"]["title"]),
226              "Delta [%]"])
227         header_str = ",".join(header) + "\n"
228     except (AttributeError, KeyError) as err:
229         logging.error("The model is invalid, missing parameter: {0}".
230                       format(err))
231         return
232
233     # Prepare data to the table:
234     tbl_dict = dict()
235     for job, builds in table["reference"]["data"].items():
236         for build in builds:
237             for tst_name, tst_data in data[job][str(build)].iteritems():
238                 tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\
239                     replace("-ndrpdr", "").replace("-pdrdisc", "").\
240                     replace("-ndrdisc", "").replace("-pdr", "").\
241                     replace("-ndr", "").\
242                     replace("1t1c", "1c").replace("2t1c", "1c").\
243                     replace("2t2c", "2c").replace("4t2c", "2c").\
244                     replace("4t4c", "4c").replace("8t4c", "4c")
245                 if "across topologies" in table["title"].lower():
246                     tst_name_mod = tst_name_mod.replace("2n1l-", "")
247                 if tbl_dict.get(tst_name_mod, None) is None:
248                     groups = re.search(REGEX_NIC, tst_data["parent"])
249                     nic = groups.group(0) if groups else ""
250                     name = "{0}-{1}".format(nic, "-".join(tst_data["name"].
251                                                           split("-")[:-1]))
252                     if "across testbeds" in table["title"].lower() or \
253                             "across topologies" in table["title"].lower():
254                         name = name.\
255                             replace("1t1c", "1c").replace("2t1c", "1c").\
256                             replace("2t2c", "2c").replace("4t2c", "2c").\
257                             replace("4t4c", "4c").replace("8t4c", "4c")
258                     tbl_dict[tst_name_mod] = {"name": name,
259                                               "ref-data": list(),
260                                               "cmp-data": list()}
261                 try:
262                     # TODO: Re-work when NDRPDRDISC tests are not used
263                     if table["include-tests"] == "MRR":
264                         tbl_dict[tst_name_mod]["ref-data"]. \
265                             append(tst_data["result"]["receive-rate"].avg)
266                     elif table["include-tests"] == "PDR":
267                         if tst_data["type"] == "PDR":
268                             tbl_dict[tst_name_mod]["ref-data"]. \
269                                 append(tst_data["throughput"]["value"])
270                         elif tst_data["type"] == "NDRPDR":
271                             tbl_dict[tst_name_mod]["ref-data"].append(
272                                 tst_data["throughput"]["PDR"]["LOWER"])
273                     elif table["include-tests"] == "NDR":
274                         if tst_data["type"] == "NDR":
275                             tbl_dict[tst_name_mod]["ref-data"]. \
276                                 append(tst_data["throughput"]["value"])
277                         elif tst_data["type"] == "NDRPDR":
278                             tbl_dict[tst_name_mod]["ref-data"].append(
279                                 tst_data["throughput"]["NDR"]["LOWER"])
280                     else:
281                         continue
282                 except TypeError:
283                     pass  # No data in output.xml for this test
284
285     for job, builds in table["compare"]["data"].items():
286         for build in builds:
287             for tst_name, tst_data in data[job][str(build)].iteritems():
288                 tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \
289                     replace("-ndrpdr", "").replace("-pdrdisc", ""). \
290                     replace("-ndrdisc", "").replace("-pdr", ""). \
291                     replace("-ndr", "").\
292                     replace("1t1c", "1c").replace("2t1c", "1c").\
293                     replace("2t2c", "2c").replace("4t2c", "2c").\
294                     replace("4t4c", "4c").replace("8t4c", "4c")
295                 if "across topologies" in table["title"].lower():
296                     tst_name_mod = tst_name_mod.replace("2n1l-", "")
297                 try:
298                     # TODO: Re-work when NDRPDRDISC tests are not used
299                     if table["include-tests"] == "MRR":
300                         tbl_dict[tst_name_mod]["cmp-data"]. \
301                             append(tst_data["result"]["receive-rate"].avg)
302                     elif table["include-tests"] == "PDR":
303                         if tst_data["type"] == "PDR":
304                             tbl_dict[tst_name_mod]["cmp-data"]. \
305                                 append(tst_data["throughput"]["value"])
306                         elif tst_data["type"] == "NDRPDR":
307                             tbl_dict[tst_name_mod]["cmp-data"].append(
308                                 tst_data["throughput"]["PDR"]["LOWER"])
309                     elif table["include-tests"] == "NDR":
310                         if tst_data["type"] == "NDR":
311                             tbl_dict[tst_name_mod]["cmp-data"]. \
312                                 append(tst_data["throughput"]["value"])
313                         elif tst_data["type"] == "NDRPDR":
314                             tbl_dict[tst_name_mod]["cmp-data"].append(
315                                 tst_data["throughput"]["NDR"]["LOWER"])
316                     else:
317                         continue
318                 except KeyError:
319                     pass
320                 except TypeError:
321                     tbl_dict.pop(tst_name_mod, None)
322     if history:
323         for item in history:
324             for job, builds in item["data"].items():
325                 for build in builds:
326                     for tst_name, tst_data in data[job][str(build)].iteritems():
327                         tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \
328                             replace("-ndrpdr", "").replace("-pdrdisc", ""). \
329                             replace("-ndrdisc", "").replace("-pdr", ""). \
330                             replace("-ndr", "").\
331                             replace("1t1c", "1c").replace("2t1c", "1c").\
332                             replace("2t2c", "2c").replace("4t2c", "2c").\
333                             replace("4t4c", "4c").replace("8t4c", "4c")
334                         if "across topologies" in table["title"].lower():
335                             tst_name_mod = tst_name_mod.replace("2n1l-", "")
336                         if tbl_dict.get(tst_name_mod, None) is None:
337                             continue
338                         if tbl_dict[tst_name_mod].get("history", None) is None:
339                             tbl_dict[tst_name_mod]["history"] = OrderedDict()
340                         if tbl_dict[tst_name_mod]["history"].get(item["title"],
341                                                              None) is None:
342                             tbl_dict[tst_name_mod]["history"][item["title"]] = \
343                                 list()
344                         try:
345                             # TODO: Re-work when NDRPDRDISC tests are not used
346                             if table["include-tests"] == "MRR":
347                                 tbl_dict[tst_name_mod]["history"][item["title"
348                                 ]].append(tst_data["result"]["receive-rate"].
349                                           avg)
350                             elif table["include-tests"] == "PDR":
351                                 if tst_data["type"] == "PDR":
352                                     tbl_dict[tst_name_mod]["history"][
353                                         item["title"]].\
354                                         append(tst_data["throughput"]["value"])
355                                 elif tst_data["type"] == "NDRPDR":
356                                     tbl_dict[tst_name_mod]["history"][item[
357                                         "title"]].append(tst_data["throughput"][
358                                         "PDR"]["LOWER"])
359                             elif table["include-tests"] == "NDR":
360                                 if tst_data["type"] == "NDR":
361                                     tbl_dict[tst_name_mod]["history"][
362                                         item["title"]].\
363                                         append(tst_data["throughput"]["value"])
364                                 elif tst_data["type"] == "NDRPDR":
365                                     tbl_dict[tst_name_mod]["history"][item[
366                                         "title"]].append(tst_data["throughput"][
367                                         "NDR"]["LOWER"])
368                             else:
369                                 continue
370                         except (TypeError, KeyError):
371                             pass
372
373     tbl_lst = list()
374     for tst_name in tbl_dict.keys():
375         item = [tbl_dict[tst_name]["name"], ]
376         if history:
377             if tbl_dict[tst_name].get("history", None) is not None:
378                 for hist_data in tbl_dict[tst_name]["history"].values():
379                     if hist_data:
380                         item.append(round(mean(hist_data) / 1000000, 2))
381                         item.append(round(stdev(hist_data) / 1000000, 2))
382                     else:
383                         item.extend([None, None])
384             else:
385                 item.extend([None, None])
386         data_t = tbl_dict[tst_name]["ref-data"]
387         if data_t:
388             item.append(round(mean(data_t) / 1000000, 2))
389             item.append(round(stdev(data_t) / 1000000, 2))
390         else:
391             item.extend([None, None])
392         data_t = tbl_dict[tst_name]["cmp-data"]
393         if data_t:
394             item.append(round(mean(data_t) / 1000000, 2))
395             item.append(round(stdev(data_t) / 1000000, 2))
396         else:
397             item.extend([None, None])
398         if item[-4] is not None and item[-2] is not None and item[-4] != 0:
399             item.append(int(relative_change(float(item[-4]), float(item[-2]))))
400         if len(item) == len(header):
401             tbl_lst.append(item)
402
403     # Sort the table according to the relative change
404     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
405
406     # Generate csv tables:
407     csv_file = "{0}.csv".format(table["output-file"])
408     with open(csv_file, "w") as file_handler:
409         file_handler.write(header_str)
410         for test in tbl_lst:
411             file_handler.write(",".join([str(item) for item in test]) + "\n")
412
413     convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
414
415
416 def table_nics_comparison(table, input_data):
417     """Generate the table(s) with algorithm: table_nics_comparison
418     specified in the specification file.
419
420     :param table: Table to generate.
421     :param input_data: Data to process.
422     :type table: pandas.Series
423     :type input_data: InputData
424     """
425
426     logging.info("  Generating the table {0} ...".
427                  format(table.get("title", "")))
428
429     # Transform the data
430     logging.info("    Creating the data set for the {0} '{1}'.".
431                  format(table.get("type", ""), table.get("title", "")))
432     data = input_data.filter_data(table, continue_on_error=True)
433
434     # Prepare the header of the tables
435     try:
436         header = ["Test case", ]
437
438         if table["include-tests"] == "MRR":
439             hdr_param = "Receive Rate"
440         else:
441             hdr_param = "Throughput"
442
443         header.extend(
444             ["{0} {1} [Mpps]".format(table["reference"]["title"], hdr_param),
445              "{0} Stdev [Mpps]".format(table["reference"]["title"]),
446              "{0} {1} [Mpps]".format(table["compare"]["title"], hdr_param),
447              "{0} Stdev [Mpps]".format(table["compare"]["title"]),
448              "Delta [%]"])
449         header_str = ",".join(header) + "\n"
450     except (AttributeError, KeyError) as err:
451         logging.error("The model is invalid, missing parameter: {0}".
452                       format(err))
453         return
454
455     # Prepare data to the table:
456     tbl_dict = dict()
457     for job, builds in table["data"].items():
458         for build in builds:
459             for tst_name, tst_data in data[job][str(build)].iteritems():
460                 tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\
461                     replace("-ndrpdr", "").replace("-pdrdisc", "").\
462                     replace("-ndrdisc", "").replace("-pdr", "").\
463                     replace("-ndr", "").\
464                     replace("1t1c", "1c").replace("2t1c", "1c").\
465                     replace("2t2c", "2c").replace("4t2c", "2c").\
466                     replace("4t4c", "4c").replace("8t4c", "4c")
467                 tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod)
468                 if tbl_dict.get(tst_name_mod, None) is None:
469                     name = "-".join(tst_data["name"].split("-")[:-1])
470                     tbl_dict[tst_name_mod] = {"name": name,
471                                               "ref-data": list(),
472                                               "cmp-data": list()}
473                 try:
474                     if table["include-tests"] == "MRR":
475                         result = tst_data["result"]["receive-rate"].avg
476                     elif table["include-tests"] == "PDR":
477                         result = tst_data["throughput"]["PDR"]["LOWER"]
478                     elif table["include-tests"] == "NDR":
479                         result = tst_data["throughput"]["NDR"]["LOWER"]
480                     else:
481                         result = None
482
483                     if result:
484                         if table["reference"]["nic"] in tst_data["tags"]:
485                             tbl_dict[tst_name_mod]["ref-data"].append(result)
486                         elif table["compare"]["nic"] in tst_data["tags"]:
487                             tbl_dict[tst_name_mod]["cmp-data"].append(result)
488                 except (TypeError, KeyError) as err:
489                     logging.debug("No data for {0}".format(tst_name))
490                     logging.debug(repr(err))
491                     # No data in output.xml for this test
492
493     tbl_lst = list()
494     for tst_name in tbl_dict.keys():
495         item = [tbl_dict[tst_name]["name"], ]
496         data_t = tbl_dict[tst_name]["ref-data"]
497         if data_t:
498             item.append(round(mean(data_t) / 1000000, 2))
499             item.append(round(stdev(data_t) / 1000000, 2))
500         else:
501             item.extend([None, None])
502         data_t = tbl_dict[tst_name]["cmp-data"]
503         if data_t:
504             item.append(round(mean(data_t) / 1000000, 2))
505             item.append(round(stdev(data_t) / 1000000, 2))
506         else:
507             item.extend([None, None])
508         if item[-4] is not None and item[-2] is not None and item[-4] != 0:
509             item.append(int(relative_change(float(item[-4]), float(item[-2]))))
510         if len(item) == len(header):
511             tbl_lst.append(item)
512
513     # Sort the table according to the relative change
514     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
515
516     # Generate csv tables:
517     csv_file = "{0}.csv".format(table["output-file"])
518     with open(csv_file, "w") as file_handler:
519         file_handler.write(header_str)
520         for test in tbl_lst:
521             file_handler.write(",".join([str(item) for item in test]) + "\n")
522
523     convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
524
525
526 def table_performance_trending_dashboard(table, input_data):
527     """Generate the table(s) with algorithm:
528     table_performance_trending_dashboard
529     specified in the specification file.
530
531     :param table: Table to generate.
532     :param input_data: Data to process.
533     :type table: pandas.Series
534     :type input_data: InputData
535     """
536
537     logging.info("  Generating the table {0} ...".
538                  format(table.get("title", "")))
539
540     # Transform the data
541     logging.info("    Creating the data set for the {0} '{1}'.".
542                  format(table.get("type", ""), table.get("title", "")))
543     data = input_data.filter_data(table, continue_on_error=True)
544
545     # Prepare the header of the tables
546     header = ["Test Case",
547               "Trend [Mpps]",
548               "Short-Term Change [%]",
549               "Long-Term Change [%]",
550               "Regressions [#]",
551               "Progressions [#]"
552               ]
553     header_str = ",".join(header) + "\n"
554
555     # Prepare data to the table:
556     tbl_dict = dict()
557     for job, builds in table["data"].items():
558         for build in builds:
559             for tst_name, tst_data in data[job][str(build)].iteritems():
560                 if tst_name.lower() in table["ignore-list"]:
561                     continue
562                 if tbl_dict.get(tst_name, None) is None:
563                     groups = re.search(REGEX_NIC, tst_data["parent"])
564                     if not groups:
565                         continue
566                     nic = groups.group(0)
567                     tbl_dict[tst_name] = {
568                         "name": "{0}-{1}".format(nic, tst_data["name"]),
569                         "data": OrderedDict()}
570                 try:
571                     tbl_dict[tst_name]["data"][str(build)] = \
572                         tst_data["result"]["receive-rate"]
573                 except (TypeError, KeyError):
574                     pass  # No data in output.xml for this test
575
576     tbl_lst = list()
577     for tst_name in tbl_dict.keys():
578         data_t = tbl_dict[tst_name]["data"]
579         if len(data_t) < 2:
580             continue
581
582         classification_lst, avgs = classify_anomalies(data_t)
583
584         win_size = min(len(data_t), table["window"])
585         long_win_size = min(len(data_t), table["long-trend-window"])
586
587         try:
588             max_long_avg = max(
589                 [x for x in avgs[-long_win_size:-win_size]
590                  if not isnan(x)])
591         except ValueError:
592             max_long_avg = nan
593         last_avg = avgs[-1]
594         avg_week_ago = avgs[max(-win_size, -len(avgs))]
595
596         if isnan(last_avg) or isnan(avg_week_ago) or avg_week_ago == 0.0:
597             rel_change_last = nan
598         else:
599             rel_change_last = round(
600                 ((last_avg - avg_week_ago) / avg_week_ago) * 100, 2)
601
602         if isnan(max_long_avg) or isnan(last_avg) or max_long_avg == 0.0:
603             rel_change_long = nan
604         else:
605             rel_change_long = round(
606                 ((last_avg - max_long_avg) / max_long_avg) * 100, 2)
607
608         if classification_lst:
609             if isnan(rel_change_last) and isnan(rel_change_long):
610                 continue
611             if (isnan(last_avg) or
612                 isnan(rel_change_last) or
613                 isnan(rel_change_long)):
614                 continue
615             tbl_lst.append(
616                 [tbl_dict[tst_name]["name"],
617                  round(last_avg / 1000000, 2),
618                  rel_change_last,
619                  rel_change_long,
620                  classification_lst[-win_size:].count("regression"),
621                  classification_lst[-win_size:].count("progression")])
622
623     tbl_lst.sort(key=lambda rel: rel[0])
624
625     tbl_sorted = list()
626     for nrr in range(table["window"], -1, -1):
627         tbl_reg = [item for item in tbl_lst if item[4] == nrr]
628         for nrp in range(table["window"], -1, -1):
629             tbl_out = [item for item in tbl_reg if item[5] == nrp]
630             tbl_out.sort(key=lambda rel: rel[2])
631             tbl_sorted.extend(tbl_out)
632
633     file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
634
635     logging.info("    Writing file: '{0}'".format(file_name))
636     with open(file_name, "w") as file_handler:
637         file_handler.write(header_str)
638         for test in tbl_sorted:
639             file_handler.write(",".join([str(item) for item in test]) + '\n')
640
641     txt_file_name = "{0}.txt".format(table["output-file"])
642     logging.info("    Writing file: '{0}'".format(txt_file_name))
643     convert_csv_to_pretty_txt(file_name, txt_file_name)
644
645
646 def _generate_url(base, testbed, test_name):
647     """Generate URL to a trending plot from the name of the test case.
648
649     :param base: The base part of URL common to all test cases.
650     :param testbed: The testbed used for testing.
651     :param test_name: The name of the test case.
652     :type base: str
653     :type testbed: str
654     :type test_name: str
655     :returns: The URL to the plot with the trending data for the given test
656         case.
657     :rtype str
658     """
659
660     url = base
661     file_name = ""
662     anchor = ".html#"
663     feature = ""
664
665     if "lbdpdk" in test_name or "lbvpp" in test_name:
666         file_name = "link_bonding"
667
668     elif "114b" in test_name and "vhost" in test_name:
669         file_name = "vts"
670
671     elif "testpmd" in test_name or "l3fwd" in test_name:
672         file_name = "dpdk"
673
674     elif "memif" in test_name:
675         file_name = "container_memif"
676         feature = "-base"
677
678     elif "srv6" in test_name:
679         file_name = "srv6"
680
681     elif "vhost" in test_name:
682         if "l2xcbase" in test_name or "l2bdbasemaclrn" in test_name:
683             file_name = "vm_vhost_l2"
684             if "114b" in test_name:
685                 feature = ""
686             elif "l2xcbase" in test_name and "x520" in test_name:
687                 feature = "-base-l2xc"
688             elif "l2bdbasemaclrn" in test_name and "x520" in test_name:
689                 feature = "-base-l2bd"
690             else:
691                 feature = "-base"
692         elif "ip4base" in test_name:
693             file_name = "vm_vhost_ip4"
694             feature = "-base"
695
696     elif "ipsec" in test_name:
697         file_name = "ipsec"
698         feature = "-base-scale"
699
700     elif "ethip4lispip" in test_name or "ethip4vxlan" in test_name:
701         file_name = "ip4_tunnels"
702         feature = "-base"
703
704     elif "ip4base" in test_name or "ip4scale" in test_name:
705         file_name = "ip4"
706         if "xl710" in test_name:
707             feature = "-base-scale-features"
708         elif "iacl" in test_name:
709             feature = "-features-iacl"
710         elif "oacl" in test_name:
711             feature = "-features-oacl"
712         elif "snat" in test_name or "cop" in test_name:
713             feature = "-features"
714         else:
715             feature = "-base-scale"
716
717     elif "ip6base" in test_name or "ip6scale" in test_name:
718         file_name = "ip6"
719         feature = "-base-scale"
720
721     elif "l2xcbase" in test_name or "l2xcscale" in test_name \
722             or "l2bdbasemaclrn" in test_name or "l2bdscale" in test_name \
723             or "l2dbbasemaclrn" in test_name or "l2dbscale" in test_name:
724         file_name = "l2"
725         if "macip" in test_name:
726             feature = "-features-macip"
727         elif "iacl" in test_name:
728             feature = "-features-iacl"
729         elif "oacl" in test_name:
730             feature = "-features-oacl"
731         else:
732             feature = "-base-scale"
733
734     if "x520" in test_name:
735         nic = "x520-"
736     elif "x710" in test_name:
737         nic = "x710-"
738     elif "xl710" in test_name:
739         nic = "xl710-"
740     elif "xxv710" in test_name:
741         nic = "xxv710-"
742     elif "vic1227" in test_name:
743         nic = "vic1227-"
744     elif "vic1385" in test_name:
745         nic = "vic1385-"
746     else:
747         nic = ""
748     anchor += nic
749
750     if "64b" in test_name:
751         framesize = "64b"
752     elif "78b" in test_name:
753         framesize = "78b"
754     elif "imix" in test_name:
755         framesize = "imix"
756     elif "9000b" in test_name:
757         framesize = "9000b"
758     elif "1518b" in test_name:
759         framesize = "1518b"
760     elif "114b" in test_name:
761         framesize = "114b"
762     else:
763         framesize = ""
764     anchor += framesize + '-'
765
766     if "1t1c" in test_name:
767         anchor += "1t1c"
768     elif "2t2c" in test_name:
769         anchor += "2t2c"
770     elif "4t4c" in test_name:
771         anchor += "4t4c"
772     elif "2t1c" in test_name:
773         anchor += "2t1c"
774     elif "4t2c" in test_name:
775         anchor += "4t2c"
776     elif "8t4c" in test_name:
777         anchor += "8t4c"
778
779     return url + file_name + '-' + testbed + '-' + nic + framesize + feature + \
780            anchor + feature
781
782
783 def table_performance_trending_dashboard_html(table, input_data):
784     """Generate the table(s) with algorithm:
785     table_performance_trending_dashboard_html specified in the specification
786     file.
787
788     :param table: Table to generate.
789     :param input_data: Data to process.
790     :type table: dict
791     :type input_data: InputData
792     """
793
794     testbed = table.get("testbed", None)
795     if testbed is None:
796         logging.error("The testbed is not defined for the table '{0}'.".
797                       format(table.get("title", "")))
798         return
799
800     logging.info("  Generating the table {0} ...".
801                  format(table.get("title", "")))
802
803     try:
804         with open(table["input-file"], 'rb') as csv_file:
805             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
806             csv_lst = [item for item in csv_content]
807     except KeyError:
808         logging.warning("The input file is not defined.")
809         return
810     except csv.Error as err:
811         logging.warning("Not possible to process the file '{0}'.\n{1}".
812                         format(table["input-file"], err))
813         return
814
815     # Table:
816     dashboard = ET.Element("table", attrib=dict(width="100%", border='0'))
817
818     # Table header:
819     tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor="#7eade7"))
820     for idx, item in enumerate(csv_lst[0]):
821         alignment = "left" if idx == 0 else "center"
822         th = ET.SubElement(tr, "th", attrib=dict(align=alignment))
823         th.text = item
824
825     # Rows:
826     colors = {"regression": ("#ffcccc", "#ff9999"),
827               "progression": ("#c6ecc6", "#9fdf9f"),
828               "normal": ("#e9f1fb", "#d4e4f7")}
829     for r_idx, row in enumerate(csv_lst[1:]):
830         if int(row[4]):
831             color = "regression"
832         elif int(row[5]):
833             color = "progression"
834         else:
835             color = "normal"
836         background = colors[color][r_idx % 2]
837         tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor=background))
838
839         # Columns:
840         for c_idx, item in enumerate(row):
841             alignment = "left" if c_idx == 0 else "center"
842             td = ET.SubElement(tr, "td", attrib=dict(align=alignment))
843             # Name:
844             if c_idx == 0:
845                 url = _generate_url("../trending/", testbed, item)
846                 ref = ET.SubElement(td, "a", attrib=dict(href=url))
847                 ref.text = item
848             else:
849                 td.text = item
850     try:
851         with open(table["output-file"], 'w') as html_file:
852             logging.info("    Writing file: '{0}'".format(table["output-file"]))
853             html_file.write(".. raw:: html\n\n\t")
854             html_file.write(ET.tostring(dashboard))
855             html_file.write("\n\t<p><br><br></p>\n")
856     except KeyError:
857         logging.warning("The output file is not defined.")
858         return
859
860
861 def table_failed_tests(table, input_data):
862     """Generate the table(s) with algorithm: table_failed_tests
863     specified in the specification file.
864
865     :param table: Table to generate.
866     :param input_data: Data to process.
867     :type table: pandas.Series
868     :type input_data: InputData
869     """
870
871     logging.info("  Generating the table {0} ...".
872                  format(table.get("title", "")))
873
874     # Transform the data
875     logging.info("    Creating the data set for the {0} '{1}'.".
876                  format(table.get("type", ""), table.get("title", "")))
877     data = input_data.filter_data(table, continue_on_error=True)
878
879     # Prepare the header of the tables
880     header = ["Test Case",
881               "Failures [#]",
882               "Last Failure [Time]",
883               "Last Failure [VPP-Build-Id]",
884               "Last Failure [CSIT-Job-Build-Id]"]
885
886     # Generate the data for the table according to the model in the table
887     # specification
888
889     now = dt.utcnow()
890     timeperiod = timedelta(int(table.get("window", 7)))
891
892     tbl_dict = dict()
893     for job, builds in table["data"].items():
894         for build in builds:
895             build = str(build)
896             for tst_name, tst_data in data[job][build].iteritems():
897                 if tst_name.lower() in table["ignore-list"]:
898                     continue
899                 if tbl_dict.get(tst_name, None) is None:
900                     groups = re.search(REGEX_NIC, tst_data["parent"])
901                     if not groups:
902                         continue
903                     nic = groups.group(0)
904                     tbl_dict[tst_name] = {
905                         "name": "{0}-{1}".format(nic, tst_data["name"]),
906                         "data": OrderedDict()}
907                 try:
908                     generated = input_data.metadata(job, build).\
909                         get("generated", "")
910                     if not generated:
911                         continue
912                     then = dt.strptime(generated, "%Y%m%d %H:%M")
913                     if (now - then) <= timeperiod:
914                         tbl_dict[tst_name]["data"][build] = (
915                             tst_data["status"],
916                             generated,
917                             input_data.metadata(job, build).get("version", ""),
918                             build)
919                 except (TypeError, KeyError) as err:
920                     logging.warning("tst_name: {} - err: {}".
921                                     format(tst_name, repr(err)))
922
923     max_fails = 0
924     tbl_lst = list()
925     for tst_data in tbl_dict.values():
926         fails_nr = 0
927         for val in tst_data["data"].values():
928             if val[0] == "FAIL":
929                 fails_nr += 1
930                 fails_last_date = val[1]
931                 fails_last_vpp = val[2]
932                 fails_last_csit = val[3]
933         if fails_nr:
934             max_fails = fails_nr if fails_nr > max_fails else max_fails
935             tbl_lst.append([tst_data["name"],
936                             fails_nr,
937                             fails_last_date,
938                             fails_last_vpp,
939                             "mrr-daily-build-{0}".format(fails_last_csit)])
940
941     tbl_lst.sort(key=lambda rel: rel[2], reverse=True)
942     tbl_sorted = list()
943     for nrf in range(max_fails, -1, -1):
944         tbl_fails = [item for item in tbl_lst if item[1] == nrf]
945         tbl_sorted.extend(tbl_fails)
946     file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
947
948     logging.info("    Writing file: '{0}'".format(file_name))
949     with open(file_name, "w") as file_handler:
950         file_handler.write(",".join(header) + "\n")
951         for test in tbl_sorted:
952             file_handler.write(",".join([str(item) for item in test]) + '\n')
953
954     txt_file_name = "{0}.txt".format(table["output-file"])
955     logging.info("    Writing file: '{0}'".format(txt_file_name))
956     convert_csv_to_pretty_txt(file_name, txt_file_name)
957
958
959 def table_failed_tests_html(table, input_data):
960     """Generate the table(s) with algorithm: table_failed_tests_html
961     specified in the specification file.
962
963     :param table: Table to generate.
964     :param input_data: Data to process.
965     :type table: pandas.Series
966     :type input_data: InputData
967     """
968
969     testbed = table.get("testbed", None)
970     if testbed is None:
971         logging.error("The testbed is not defined for the table '{0}'.".
972                       format(table.get("title", "")))
973         return
974
975     logging.info("  Generating the table {0} ...".
976                  format(table.get("title", "")))
977
978     try:
979         with open(table["input-file"], 'rb') as csv_file:
980             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
981             csv_lst = [item for item in csv_content]
982     except KeyError:
983         logging.warning("The input file is not defined.")
984         return
985     except csv.Error as err:
986         logging.warning("Not possible to process the file '{0}'.\n{1}".
987                         format(table["input-file"], err))
988         return
989
990     # Table:
991     failed_tests = ET.Element("table", attrib=dict(width="100%", border='0'))
992
993     # Table header:
994     tr = ET.SubElement(failed_tests, "tr", attrib=dict(bgcolor="#7eade7"))
995     for idx, item in enumerate(csv_lst[0]):
996         alignment = "left" if idx == 0 else "center"
997         th = ET.SubElement(tr, "th", attrib=dict(align=alignment))
998         th.text = item
999
1000     # Rows:
1001     colors = ("#e9f1fb", "#d4e4f7")
1002     for r_idx, row in enumerate(csv_lst[1:]):
1003         background = colors[r_idx % 2]
1004         tr = ET.SubElement(failed_tests, "tr", attrib=dict(bgcolor=background))
1005
1006         # Columns:
1007         for c_idx, item in enumerate(row):
1008             alignment = "left" if c_idx == 0 else "center"
1009             td = ET.SubElement(tr, "td", attrib=dict(align=alignment))
1010             # Name:
1011             if c_idx == 0:
1012                 url = _generate_url("../trending/", testbed, item)
1013                 ref = ET.SubElement(td, "a", attrib=dict(href=url))
1014                 ref.text = item
1015             else:
1016                 td.text = item
1017     try:
1018         with open(table["output-file"], 'w') as html_file:
1019             logging.info("    Writing file: '{0}'".format(table["output-file"]))
1020             html_file.write(".. raw:: html\n\n\t")
1021             html_file.write(ET.tostring(failed_tests))
1022             html_file.write("\n\t<p><br><br></p>\n")
1023     except KeyError:
1024         logging.warning("The output file is not defined.")
1025         return

©2016 FD.io a Linux Foundation Collaborative Project. All Rights Reserved.
Linux Foundation is a registered trademark of The Linux Foundation. Linux is a registered trademark of Linus Torvalds.
Please see our privacy policy and terms of use.