Trending: Alerts
[csit.git] / resources / tools / presentation / generator_tables.py
1 # Copyright (c) 2019 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
5 #
6 #     http://www.apache.org/licenses/LICENSE-2.0
7 #
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
13
14 """Algorithms to generate tables.
15 """
16
17
18 import logging
19 import csv
20 import re
21
22 from string import replace
23 from collections import OrderedDict
24 from numpy import nan, isnan
25 from xml.etree import ElementTree as ET
26 from datetime import datetime as dt
27 from datetime import timedelta
28
29 from utils import mean, stdev, relative_change, classify_anomalies, \
30     convert_csv_to_pretty_txt
31
32
33 REGEX_NIC = re.compile(r'\d*ge\dp\d\D*\d*')
34
35
36 def generate_tables(spec, data):
37     """Generate all tables specified in the specification file.
38
39     :param spec: Specification read from the specification file.
40     :param data: Data to process.
41     :type spec: Specification
42     :type data: InputData
43     """
44
45     logging.info("Generating the tables ...")
46     for table in spec.tables:
47         try:
48             eval(table["algorithm"])(table, data)
49         except NameError as err:
50             logging.error("Probably algorithm '{alg}' is not defined: {err}".
51                           format(alg=table["algorithm"], err=repr(err)))
52     logging.info("Done.")
53
54
55 def table_details(table, input_data):
56     """Generate the table(s) with algorithm: table_detailed_test_results
57     specified in the specification file.
58
59     :param table: Table to generate.
60     :param input_data: Data to process.
61     :type table: pandas.Series
62     :type input_data: InputData
63     """
64
65     logging.info("  Generating the table {0} ...".
66                  format(table.get("title", "")))
67
68     # Transform the data
69     logging.info("    Creating the data set for the {0} '{1}'.".
70                  format(table.get("type", ""), table.get("title", "")))
71     data = input_data.filter_data(table)
72
73     # Prepare the header of the tables
74     header = list()
75     for column in table["columns"]:
76         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
77
78     # Generate the data for the table according to the model in the table
79     # specification
80     job = table["data"].keys()[0]
81     build = str(table["data"][job][0])
82     try:
83         suites = input_data.suites(job, build)
84     except KeyError:
85         logging.error("    No data available. The table will not be generated.")
86         return
87
88     for suite_longname, suite in suites.iteritems():
89         # Generate data
90         suite_name = suite["name"]
91         table_lst = list()
92         for test in data[job][build].keys():
93             if data[job][build][test]["parent"] in suite_name:
94                 row_lst = list()
95                 for column in table["columns"]:
96                     try:
97                         col_data = str(data[job][build][test][column["data"].
98                                        split(" ")[1]]).replace('"', '""')
99                         if column["data"].split(" ")[1] in ("conf-history",
100                                                             "show-run"):
101                             col_data = replace(col_data, " |br| ", "",
102                                                maxreplace=1)
103                             col_data = " |prein| {0} |preout| ".\
104                                 format(col_data[:-5])
105                         row_lst.append('"{0}"'.format(col_data))
106                     except KeyError:
107                         row_lst.append("No data")
108                 table_lst.append(row_lst)
109
110         # Write the data to file
111         if table_lst:
112             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
113                                             table["output-file-ext"])
114             logging.info("      Writing file: '{}'".format(file_name))
115             with open(file_name, "w") as file_handler:
116                 file_handler.write(",".join(header) + "\n")
117                 for item in table_lst:
118                     file_handler.write(",".join(item) + "\n")
119
120     logging.info("  Done.")
121
122
123 def table_merged_details(table, input_data):
124     """Generate the table(s) with algorithm: table_merged_details
125     specified in the specification file.
126
127     :param table: Table to generate.
128     :param input_data: Data to process.
129     :type table: pandas.Series
130     :type input_data: InputData
131     """
132
133     logging.info("  Generating the table {0} ...".
134                  format(table.get("title", "")))
135
136     # Transform the data
137     logging.info("    Creating the data set for the {0} '{1}'.".
138                  format(table.get("type", ""), table.get("title", "")))
139     data = input_data.filter_data(table)
140     data = input_data.merge_data(data)
141     data.sort_index(inplace=True)
142
143     logging.info("    Creating the data set for the {0} '{1}'.".
144                  format(table.get("type", ""), table.get("title", "")))
145     suites = input_data.filter_data(table, data_set="suites")
146     suites = input_data.merge_data(suites)
147
148     # Prepare the header of the tables
149     header = list()
150     for column in table["columns"]:
151         header.append('"{0}"'.format(str(column["title"]).replace('"', '""')))
152
153     for _, suite in suites.iteritems():
154         # Generate data
155         suite_name = suite["name"]
156         table_lst = list()
157         for test in data.keys():
158             if data[test]["parent"] in suite_name:
159                 row_lst = list()
160                 for column in table["columns"]:
161                     try:
162                         col_data = str(data[test][column["data"].
163                                        split(" ")[1]]).replace('"', '""')
164                         if column["data"].split(" ")[1] in ("conf-history",
165                                                             "show-run"):
166                             col_data = replace(col_data, " |br| ", "",
167                                                maxreplace=1)
168                             col_data = " |prein| {0} |preout| ".\
169                                 format(col_data[:-5])
170                         row_lst.append('"{0}"'.format(col_data))
171                     except KeyError:
172                         row_lst.append("No data")
173                 table_lst.append(row_lst)
174
175         # Write the data to file
176         if table_lst:
177             file_name = "{0}_{1}{2}".format(table["output-file"], suite_name,
178                                             table["output-file-ext"])
179             logging.info("      Writing file: '{}'".format(file_name))
180             with open(file_name, "w") as file_handler:
181                 file_handler.write(",".join(header) + "\n")
182                 for item in table_lst:
183                     file_handler.write(",".join(item) + "\n")
184
185     logging.info("  Done.")
186
187
188 def table_performance_comparison(table, input_data):
189     """Generate the table(s) with algorithm: table_performance_comparison
190     specified in the specification file.
191
192     :param table: Table to generate.
193     :param input_data: Data to process.
194     :type table: pandas.Series
195     :type input_data: InputData
196     """
197
198     logging.info("  Generating the table {0} ...".
199                  format(table.get("title", "")))
200
201     # Transform the data
202     logging.info("    Creating the data set for the {0} '{1}'.".
203                  format(table.get("type", ""), table.get("title", "")))
204     data = input_data.filter_data(table, continue_on_error=True)
205
206     # Prepare the header of the tables
207     try:
208         header = ["Test case", ]
209
210         if table["include-tests"] == "MRR":
211             hdr_param = "Receive Rate"
212         else:
213             hdr_param = "Throughput"
214
215         history = table.get("history", None)
216         if history:
217             for item in history:
218                 header.extend(
219                     ["{0} {1} [Mpps]".format(item["title"], hdr_param),
220                      "{0} Stdev [Mpps]".format(item["title"])])
221         header.extend(
222             ["{0} {1} [Mpps]".format(table["reference"]["title"], hdr_param),
223              "{0} Stdev [Mpps]".format(table["reference"]["title"]),
224              "{0} {1} [Mpps]".format(table["compare"]["title"], hdr_param),
225              "{0} Stdev [Mpps]".format(table["compare"]["title"]),
226              "Delta [%]"])
227         header_str = ",".join(header) + "\n"
228     except (AttributeError, KeyError) as err:
229         logging.error("The model is invalid, missing parameter: {0}".
230                       format(err))
231         return
232
233     # Prepare data to the table:
234     tbl_dict = dict()
235     for job, builds in table["reference"]["data"].items():
236         for build in builds:
237             for tst_name, tst_data in data[job][str(build)].iteritems():
238                 tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\
239                     replace("-ndrpdr", "").replace("-pdrdisc", "").\
240                     replace("-ndrdisc", "").replace("-pdr", "").\
241                     replace("-ndr", "").\
242                     replace("1t1c", "1c").replace("2t1c", "1c").\
243                     replace("2t2c", "2c").replace("4t2c", "2c").\
244                     replace("4t4c", "4c").replace("8t4c", "4c")
245                 if "across topologies" in table["title"].lower():
246                     tst_name_mod = tst_name_mod.replace("2n1l-", "")
247                 if tbl_dict.get(tst_name_mod, None) is None:
248                     groups = re.search(REGEX_NIC, tst_data["parent"])
249                     nic = groups.group(0) if groups else ""
250                     name = "{0}-{1}".format(nic, "-".join(tst_data["name"].
251                                                           split("-")[:-1]))
252                     if "across testbeds" in table["title"].lower() or \
253                             "across topologies" in table["title"].lower():
254                         name = name.\
255                             replace("1t1c", "1c").replace("2t1c", "1c").\
256                             replace("2t2c", "2c").replace("4t2c", "2c").\
257                             replace("4t4c", "4c").replace("8t4c", "4c")
258                     tbl_dict[tst_name_mod] = {"name": name,
259                                               "ref-data": list(),
260                                               "cmp-data": list()}
261                 try:
262                     # TODO: Re-work when NDRPDRDISC tests are not used
263                     if table["include-tests"] == "MRR":
264                         tbl_dict[tst_name_mod]["ref-data"]. \
265                             append(tst_data["result"]["receive-rate"].avg)
266                     elif table["include-tests"] == "PDR":
267                         if tst_data["type"] == "PDR":
268                             tbl_dict[tst_name_mod]["ref-data"]. \
269                                 append(tst_data["throughput"]["value"])
270                         elif tst_data["type"] == "NDRPDR":
271                             tbl_dict[tst_name_mod]["ref-data"].append(
272                                 tst_data["throughput"]["PDR"]["LOWER"])
273                     elif table["include-tests"] == "NDR":
274                         if tst_data["type"] == "NDR":
275                             tbl_dict[tst_name_mod]["ref-data"]. \
276                                 append(tst_data["throughput"]["value"])
277                         elif tst_data["type"] == "NDRPDR":
278                             tbl_dict[tst_name_mod]["ref-data"].append(
279                                 tst_data["throughput"]["NDR"]["LOWER"])
280                     else:
281                         continue
282                 except TypeError:
283                     pass  # No data in output.xml for this test
284
285     for job, builds in table["compare"]["data"].items():
286         for build in builds:
287             for tst_name, tst_data in data[job][str(build)].iteritems():
288                 tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \
289                     replace("-ndrpdr", "").replace("-pdrdisc", ""). \
290                     replace("-ndrdisc", "").replace("-pdr", ""). \
291                     replace("-ndr", "").\
292                     replace("1t1c", "1c").replace("2t1c", "1c").\
293                     replace("2t2c", "2c").replace("4t2c", "2c").\
294                     replace("4t4c", "4c").replace("8t4c", "4c")
295                 if "across topologies" in table["title"].lower():
296                     tst_name_mod = tst_name_mod.replace("2n1l-", "")
297                 try:
298                     # TODO: Re-work when NDRPDRDISC tests are not used
299                     if table["include-tests"] == "MRR":
300                         tbl_dict[tst_name_mod]["cmp-data"]. \
301                             append(tst_data["result"]["receive-rate"].avg)
302                     elif table["include-tests"] == "PDR":
303                         if tst_data["type"] == "PDR":
304                             tbl_dict[tst_name_mod]["cmp-data"]. \
305                                 append(tst_data["throughput"]["value"])
306                         elif tst_data["type"] == "NDRPDR":
307                             tbl_dict[tst_name_mod]["cmp-data"].append(
308                                 tst_data["throughput"]["PDR"]["LOWER"])
309                     elif table["include-tests"] == "NDR":
310                         if tst_data["type"] == "NDR":
311                             tbl_dict[tst_name_mod]["cmp-data"]. \
312                                 append(tst_data["throughput"]["value"])
313                         elif tst_data["type"] == "NDRPDR":
314                             tbl_dict[tst_name_mod]["cmp-data"].append(
315                                 tst_data["throughput"]["NDR"]["LOWER"])
316                     else:
317                         continue
318                 except KeyError:
319                     pass
320                 except TypeError:
321                     tbl_dict.pop(tst_name_mod, None)
322     if history:
323         for item in history:
324             for job, builds in item["data"].items():
325                 for build in builds:
326                     for tst_name, tst_data in data[job][str(build)].iteritems():
327                         tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \
328                             replace("-ndrpdr", "").replace("-pdrdisc", ""). \
329                             replace("-ndrdisc", "").replace("-pdr", ""). \
330                             replace("-ndr", "").\
331                             replace("1t1c", "1c").replace("2t1c", "1c").\
332                             replace("2t2c", "2c").replace("4t2c", "2c").\
333                             replace("4t4c", "4c").replace("8t4c", "4c")
334                         if "across topologies" in table["title"].lower():
335                             tst_name_mod = tst_name_mod.replace("2n1l-", "")
336                         if tbl_dict.get(tst_name_mod, None) is None:
337                             continue
338                         if tbl_dict[tst_name_mod].get("history", None) is None:
339                             tbl_dict[tst_name_mod]["history"] = OrderedDict()
340                         if tbl_dict[tst_name_mod]["history"].get(item["title"],
341                                                              None) is None:
342                             tbl_dict[tst_name_mod]["history"][item["title"]] = \
343                                 list()
344                         try:
345                             # TODO: Re-work when NDRPDRDISC tests are not used
346                             if table["include-tests"] == "MRR":
347                                 tbl_dict[tst_name_mod]["history"][item["title"
348                                 ]].append(tst_data["result"]["receive-rate"].
349                                           avg)
350                             elif table["include-tests"] == "PDR":
351                                 if tst_data["type"] == "PDR":
352                                     tbl_dict[tst_name_mod]["history"][
353                                         item["title"]].\
354                                         append(tst_data["throughput"]["value"])
355                                 elif tst_data["type"] == "NDRPDR":
356                                     tbl_dict[tst_name_mod]["history"][item[
357                                         "title"]].append(tst_data["throughput"][
358                                         "PDR"]["LOWER"])
359                             elif table["include-tests"] == "NDR":
360                                 if tst_data["type"] == "NDR":
361                                     tbl_dict[tst_name_mod]["history"][
362                                         item["title"]].\
363                                         append(tst_data["throughput"]["value"])
364                                 elif tst_data["type"] == "NDRPDR":
365                                     tbl_dict[tst_name_mod]["history"][item[
366                                         "title"]].append(tst_data["throughput"][
367                                         "NDR"]["LOWER"])
368                             else:
369                                 continue
370                         except (TypeError, KeyError):
371                             pass
372
373     tbl_lst = list()
374     for tst_name in tbl_dict.keys():
375         item = [tbl_dict[tst_name]["name"], ]
376         if history:
377             if tbl_dict[tst_name].get("history", None) is not None:
378                 for hist_data in tbl_dict[tst_name]["history"].values():
379                     if hist_data:
380                         item.append(round(mean(hist_data) / 1000000, 2))
381                         item.append(round(stdev(hist_data) / 1000000, 2))
382                     else:
383                         item.extend([None, None])
384             else:
385                 item.extend([None, None])
386         data_t = tbl_dict[tst_name]["ref-data"]
387         if data_t:
388             item.append(round(mean(data_t) / 1000000, 2))
389             item.append(round(stdev(data_t) / 1000000, 2))
390         else:
391             item.extend([None, None])
392         data_t = tbl_dict[tst_name]["cmp-data"]
393         if data_t:
394             item.append(round(mean(data_t) / 1000000, 2))
395             item.append(round(stdev(data_t) / 1000000, 2))
396         else:
397             item.extend([None, None])
398         if item[-4] is not None and item[-2] is not None and item[-4] != 0:
399             item.append(int(relative_change(float(item[-4]), float(item[-2]))))
400         if len(item) == len(header):
401             tbl_lst.append(item)
402
403     # Sort the table according to the relative change
404     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
405
406     # Generate csv tables:
407     csv_file = "{0}.csv".format(table["output-file"])
408     with open(csv_file, "w") as file_handler:
409         file_handler.write(header_str)
410         for test in tbl_lst:
411             file_handler.write(",".join([str(item) for item in test]) + "\n")
412
413     convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
414
415
416 def table_nics_comparison(table, input_data):
417     """Generate the table(s) with algorithm: table_nics_comparison
418     specified in the specification file.
419
420     :param table: Table to generate.
421     :param input_data: Data to process.
422     :type table: pandas.Series
423     :type input_data: InputData
424     """
425
426     logging.info("  Generating the table {0} ...".
427                  format(table.get("title", "")))
428
429     # Transform the data
430     logging.info("    Creating the data set for the {0} '{1}'.".
431                  format(table.get("type", ""), table.get("title", "")))
432     data = input_data.filter_data(table, continue_on_error=True)
433
434     # Prepare the header of the tables
435     try:
436         header = ["Test case", ]
437
438         if table["include-tests"] == "MRR":
439             hdr_param = "Receive Rate"
440         else:
441             hdr_param = "Throughput"
442
443         header.extend(
444             ["{0} {1} [Mpps]".format(table["reference"]["title"], hdr_param),
445              "{0} Stdev [Mpps]".format(table["reference"]["title"]),
446              "{0} {1} [Mpps]".format(table["compare"]["title"], hdr_param),
447              "{0} Stdev [Mpps]".format(table["compare"]["title"]),
448              "Delta [%]"])
449         header_str = ",".join(header) + "\n"
450     except (AttributeError, KeyError) as err:
451         logging.error("The model is invalid, missing parameter: {0}".
452                       format(err))
453         return
454
455     # Prepare data to the table:
456     tbl_dict = dict()
457     for job, builds in table["data"].items():
458         for build in builds:
459             for tst_name, tst_data in data[job][str(build)].iteritems():
460                 tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\
461                     replace("-ndrpdr", "").replace("-pdrdisc", "").\
462                     replace("-ndrdisc", "").replace("-pdr", "").\
463                     replace("-ndr", "").\
464                     replace("1t1c", "1c").replace("2t1c", "1c").\
465                     replace("2t2c", "2c").replace("4t2c", "2c").\
466                     replace("4t4c", "4c").replace("8t4c", "4c")
467                 tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod)
468                 if tbl_dict.get(tst_name_mod, None) is None:
469                     name = "-".join(tst_data["name"].split("-")[:-1])
470                     tbl_dict[tst_name_mod] = {"name": name,
471                                               "ref-data": list(),
472                                               "cmp-data": list()}
473                 try:
474                     if table["include-tests"] == "MRR":
475                         result = tst_data["result"]["receive-rate"].avg
476                     elif table["include-tests"] == "PDR":
477                         result = tst_data["throughput"]["PDR"]["LOWER"]
478                     elif table["include-tests"] == "NDR":
479                         result = tst_data["throughput"]["NDR"]["LOWER"]
480                     else:
481                         result = None
482
483                     if result:
484                         if table["reference"]["nic"] in tst_data["tags"]:
485                             tbl_dict[tst_name_mod]["ref-data"].append(result)
486                         elif table["compare"]["nic"] in tst_data["tags"]:
487                             tbl_dict[tst_name_mod]["cmp-data"].append(result)
488                 except (TypeError, KeyError) as err:
489                     logging.debug("No data for {0}".format(tst_name))
490                     logging.debug(repr(err))
491                     # No data in output.xml for this test
492
493     tbl_lst = list()
494     for tst_name in tbl_dict.keys():
495         item = [tbl_dict[tst_name]["name"], ]
496         data_t = tbl_dict[tst_name]["ref-data"]
497         if data_t:
498             item.append(round(mean(data_t) / 1000000, 2))
499             item.append(round(stdev(data_t) / 1000000, 2))
500         else:
501             item.extend([None, None])
502         data_t = tbl_dict[tst_name]["cmp-data"]
503         if data_t:
504             item.append(round(mean(data_t) / 1000000, 2))
505             item.append(round(stdev(data_t) / 1000000, 2))
506         else:
507             item.extend([None, None])
508         if item[-4] is not None and item[-2] is not None and item[-4] != 0:
509             item.append(int(relative_change(float(item[-4]), float(item[-2]))))
510         if len(item) == len(header):
511             tbl_lst.append(item)
512
513     # Sort the table according to the relative change
514     tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
515
516     # Generate csv tables:
517     csv_file = "{0}.csv".format(table["output-file"])
518     with open(csv_file, "w") as file_handler:
519         file_handler.write(header_str)
520         for test in tbl_lst:
521             file_handler.write(",".join([str(item) for item in test]) + "\n")
522
523     convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
524
525
526 def table_performance_trending_dashboard(table, input_data):
527     """Generate the table(s) with algorithm:
528     table_performance_trending_dashboard
529     specified in the specification file.
530
531     :param table: Table to generate.
532     :param input_data: Data to process.
533     :type table: pandas.Series
534     :type input_data: InputData
535     """
536
537     logging.info("  Generating the table {0} ...".
538                  format(table.get("title", "")))
539
540     # Transform the data
541     logging.info("    Creating the data set for the {0} '{1}'.".
542                  format(table.get("type", ""), table.get("title", "")))
543     data = input_data.filter_data(table, continue_on_error=True)
544
545     # Prepare the header of the tables
546     header = ["Test Case",
547               "Trend [Mpps]",
548               "Short-Term Change [%]",
549               "Long-Term Change [%]",
550               "Regressions [#]",
551               "Progressions [#]"
552               ]
553     header_str = ",".join(header) + "\n"
554
555     # Prepare data to the table:
556     tbl_dict = dict()
557     for job, builds in table["data"].items():
558         for build in builds:
559             for tst_name, tst_data in data[job][str(build)].iteritems():
560                 if tst_name.lower() in table["ignore-list"]:
561                     continue
562                 if tbl_dict.get(tst_name, None) is None:
563                     groups = re.search(REGEX_NIC, tst_data["parent"])
564                     if not groups:
565                         continue
566                     nic = groups.group(0)
567                     tbl_dict[tst_name] = {
568                         "name": "{0}-{1}".format(nic, tst_data["name"]),
569                         "data": OrderedDict()}
570                 try:
571                     tbl_dict[tst_name]["data"][str(build)] = \
572                         tst_data["result"]["receive-rate"]
573                 except (TypeError, KeyError):
574                     pass  # No data in output.xml for this test
575
576     tbl_lst = list()
577     for tst_name in tbl_dict.keys():
578         data_t = tbl_dict[tst_name]["data"]
579         if len(data_t) < 2:
580             continue
581
582         classification_lst, avgs = classify_anomalies(data_t)
583
584         win_size = min(len(data_t), table["window"])
585         long_win_size = min(len(data_t), table["long-trend-window"])
586
587         try:
588             max_long_avg = max(
589                 [x for x in avgs[-long_win_size:-win_size]
590                  if not isnan(x)])
591         except ValueError:
592             max_long_avg = nan
593         last_avg = avgs[-1]
594         avg_week_ago = avgs[max(-win_size, -len(avgs))]
595
596         if isnan(last_avg) or isnan(avg_week_ago) or avg_week_ago == 0.0:
597             rel_change_last = nan
598         else:
599             rel_change_last = round(
600                 ((last_avg - avg_week_ago) / avg_week_ago) * 100, 2)
601
602         if isnan(max_long_avg) or isnan(last_avg) or max_long_avg == 0.0:
603             rel_change_long = nan
604         else:
605             rel_change_long = round(
606                 ((last_avg - max_long_avg) / max_long_avg) * 100, 2)
607
608         if classification_lst:
609             if isnan(rel_change_last) and isnan(rel_change_long):
610                 continue
611             if (isnan(last_avg) or
612                 isnan(rel_change_last) or
613                 isnan(rel_change_long)):
614                 continue
615             tbl_lst.append(
616                 [tbl_dict[tst_name]["name"],
617                  round(last_avg / 1000000, 2),
618                  rel_change_last,
619                  rel_change_long,
620                  classification_lst[-win_size:].count("regression"),
621                  classification_lst[-win_size:].count("progression")])
622
623     tbl_lst.sort(key=lambda rel: rel[0])
624
625     tbl_sorted = list()
626     for nrr in range(table["window"], -1, -1):
627         tbl_reg = [item for item in tbl_lst if item[4] == nrr]
628         for nrp in range(table["window"], -1, -1):
629             tbl_out = [item for item in tbl_reg if item[5] == nrp]
630             tbl_out.sort(key=lambda rel: rel[2])
631             tbl_sorted.extend(tbl_out)
632
633     file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
634
635     logging.info("    Writing file: '{0}'".format(file_name))
636     with open(file_name, "w") as file_handler:
637         file_handler.write(header_str)
638         for test in tbl_sorted:
639             file_handler.write(",".join([str(item) for item in test]) + '\n')
640
641     txt_file_name = "{0}.txt".format(table["output-file"])
642     logging.info("    Writing file: '{0}'".format(txt_file_name))
643     convert_csv_to_pretty_txt(file_name, txt_file_name)
644
645
646 def _generate_url(base, testbed, test_name):
647     """Generate URL to a trending plot from the name of the test case.
648
649     :param base: The base part of URL common to all test cases.
650     :param testbed: The testbed used for testing.
651     :param test_name: The name of the test case.
652     :type base: str
653     :type testbed: str
654     :type test_name: str
655     :returns: The URL to the plot with the trending data for the given test
656         case.
657     :rtype str
658     """
659
660     url = base
661     file_name = ""
662     anchor = ".html#"
663     feature = ""
664
665     if "lbdpdk" in test_name or "lbvpp" in test_name:
666         file_name = "link_bonding"
667
668     elif "114b" in test_name and "vhost" in test_name:
669         file_name = "vts"
670
671     elif "testpmd" in test_name or "l3fwd" in test_name:
672         file_name = "dpdk"
673
674     elif "memif" in test_name:
675         file_name = "container_memif"
676         feature = "-base"
677
678     elif "srv6" in test_name:
679         file_name = "srv6"
680
681     elif "vhost" in test_name:
682         if "l2xcbase" in test_name or "l2bdbasemaclrn" in test_name:
683             file_name = "vm_vhost_l2"
684             if "114b" in test_name:
685                 feature = ""
686             elif "l2xcbase" in test_name and "x520" in test_name:
687                 feature = "-base-l2xc"
688             elif "l2bdbasemaclrn" in test_name and "x520" in test_name:
689                 feature = "-base-l2bd"
690             else:
691                 feature = "-base"
692         elif "ip4base" in test_name:
693             file_name = "vm_vhost_ip4"
694             feature = "-base"
695
696     elif "ipsec" in test_name:
697         file_name = "ipsec"
698         feature = "-base-scale"
699
700     elif "ethip4lispip" in test_name or "ethip4vxlan" in test_name:
701         file_name = "ip4_tunnels"
702         feature = "-base"
703
704     elif "ip4base" in test_name or "ip4scale" in test_name:
705         file_name = "ip4"
706         if "xl710" in test_name:
707             feature = "-base-scale-features"
708         elif "iacl" in test_name:
709             feature = "-features-iacl"
710         elif "oacl" in test_name:
711             feature = "-features-oacl"
712         elif "snat" in test_name or "cop" in test_name:
713             feature = "-features"
714         else:
715             feature = "-base-scale"
716
717     elif "ip6base" in test_name or "ip6scale" in test_name:
718         file_name = "ip6"
719         feature = "-base-scale"
720
721     elif "l2xcbase" in test_name or "l2xcscale" in test_name \
722             or "l2bdbasemaclrn" in test_name or "l2bdscale" in test_name \
723             or "l2dbbasemaclrn" in test_name or "l2dbscale" in test_name:
724         file_name = "l2"
725         if "macip" in test_name:
726             feature = "-features-macip"
727         elif "iacl" in test_name:
728             feature = "-features-iacl"
729         elif "oacl" in test_name:
730             feature = "-features-oacl"
731         else:
732             feature = "-base-scale"
733
734     if "x520" in test_name:
735         nic = "x520-"
736     elif "x710" in test_name:
737         nic = "x710-"
738     elif "xl710" in test_name:
739         nic = "xl710-"
740     elif "xxv710" in test_name:
741         nic = "xxv710-"
742     elif "vic1227" in test_name:
743         nic = "vic1227-"
744     elif "vic1385" in test_name:
745         nic = "vic1385-"
746     else:
747         nic = ""
748     anchor += nic
749
750     if "64b" in test_name:
751         framesize = "64b"
752     elif "78b" in test_name:
753         framesize = "78b"
754     elif "imix" in test_name:
755         framesize = "imix"
756     elif "9000b" in test_name:
757         framesize = "9000b"
758     elif "1518b" in test_name:
759         framesize = "1518b"
760     elif "114b" in test_name:
761         framesize = "114b"
762     else:
763         framesize = ""
764     anchor += framesize + '-'
765
766     if "1t1c" in test_name:
767         anchor += "1t1c"
768     elif "2t2c" in test_name:
769         anchor += "2t2c"
770     elif "4t4c" in test_name:
771         anchor += "4t4c"
772     elif "2t1c" in test_name:
773         anchor += "2t1c"
774     elif "4t2c" in test_name:
775         anchor += "4t2c"
776     elif "8t4c" in test_name:
777         anchor += "8t4c"
778
779     return url + file_name + '-' + testbed + '-' + nic + framesize + feature + \
780            anchor + feature
781
782
783 def table_performance_trending_dashboard_html(table, input_data):
784     """Generate the table(s) with algorithm:
785     table_performance_trending_dashboard_html specified in the specification
786     file.
787
788     :param table: Table to generate.
789     :param input_data: Data to process.
790     :type table: dict
791     :type input_data: InputData
792     """
793
794     testbed = table.get("testbed", None)
795     if testbed is None:
796         logging.error("The testbed is not defined for the table '{0}'.".
797                       format(table.get("title", "")))
798         return
799
800     logging.info("  Generating the table {0} ...".
801                  format(table.get("title", "")))
802
803     try:
804         with open(table["input-file"], 'rb') as csv_file:
805             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
806             csv_lst = [item for item in csv_content]
807     except KeyError:
808         logging.warning("The input file is not defined.")
809         return
810     except csv.Error as err:
811         logging.warning("Not possible to process the file '{0}'.\n{1}".
812                         format(table["input-file"], err))
813         return
814
815     # Table:
816     dashboard = ET.Element("table", attrib=dict(width="100%", border='0'))
817
818     # Table header:
819     tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor="#7eade7"))
820     for idx, item in enumerate(csv_lst[0]):
821         alignment = "left" if idx == 0 else "center"
822         th = ET.SubElement(tr, "th", attrib=dict(align=alignment))
823         th.text = item
824
825     # Rows:
826     colors = {"regression": ("#ffcccc", "#ff9999"),
827               "progression": ("#c6ecc6", "#9fdf9f"),
828               "normal": ("#e9f1fb", "#d4e4f7")}
829     for r_idx, row in enumerate(csv_lst[1:]):
830         if int(row[4]):
831             color = "regression"
832         elif int(row[5]):
833             color = "progression"
834         else:
835             color = "normal"
836         background = colors[color][r_idx % 2]
837         tr = ET.SubElement(dashboard, "tr", attrib=dict(bgcolor=background))
838
839         # Columns:
840         for c_idx, item in enumerate(row):
841             alignment = "left" if c_idx == 0 else "center"
842             td = ET.SubElement(tr, "td", attrib=dict(align=alignment))
843             # Name:
844             if c_idx == 0:
845                 url = _generate_url("../trending/", testbed, item)
846                 ref = ET.SubElement(td, "a", attrib=dict(href=url))
847                 ref.text = item
848             else:
849                 td.text = item
850     try:
851         with open(table["output-file"], 'w') as html_file:
852             logging.info("    Writing file: '{0}'".format(table["output-file"]))
853             html_file.write(".. raw:: html\n\n\t")
854             html_file.write(ET.tostring(dashboard))
855             html_file.write("\n\t<p><br><br></p>\n")
856     except KeyError:
857         logging.warning("The output file is not defined.")
858         return
859
860
861 def table_last_failed_tests(table, input_data):
862     """Generate the table(s) with algorithm: table_last_failed_tests
863     specified in the specification file.
864
865     :param table: Table to generate.
866     :param input_data: Data to process.
867     :type table: pandas.Series
868     :type input_data: InputData
869     """
870
871     logging.info("  Generating the table {0} ...".
872                  format(table.get("title", "")))
873
874     # Transform the data
875     logging.info("    Creating the data set for the {0} '{1}'.".
876                  format(table.get("type", ""), table.get("title", "")))
877     data = input_data.filter_data(table, continue_on_error=True)
878
879     if data is None or data.empty:
880         logging.warn("    No data for the {0} '{1}'.".
881                      format(table.get("type", ""), table.get("title", "")))
882         return
883
884     tbl_list = list()
885     for job, builds in table["data"].items():
886         for build in builds:
887             build = str(build)
888             tbl_list.append(build)
889             tbl_list.append(input_data.metadata(job, build).get("version", ""))
890             for tst_name, tst_data in data[job][build].iteritems():
891                 if tst_data["status"] != "FAIL":
892                     continue
893                 groups = re.search(REGEX_NIC, tst_data["parent"])
894                 if not groups:
895                     continue
896                 nic = groups.group(0)
897                 tbl_list.append("{0}-{1}".format(nic, tst_data["name"]))
898
899     file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
900     logging.info("    Writing file: '{0}'".format(file_name))
901     with open(file_name, "w") as file_handler:
902         for test in tbl_list:
903             file_handler.write(test + '\n')
904
905
906 def table_failed_tests(table, input_data):
907     """Generate the table(s) with algorithm: table_failed_tests
908     specified in the specification file.
909
910     :param table: Table to generate.
911     :param input_data: Data to process.
912     :type table: pandas.Series
913     :type input_data: InputData
914     """
915
916     logging.info("  Generating the table {0} ...".
917                  format(table.get("title", "")))
918
919     # Transform the data
920     logging.info("    Creating the data set for the {0} '{1}'.".
921                  format(table.get("type", ""), table.get("title", "")))
922     data = input_data.filter_data(table, continue_on_error=True)
923
924     # Prepare the header of the tables
925     header = ["Test Case",
926               "Failures [#]",
927               "Last Failure [Time]",
928               "Last Failure [VPP-Build-Id]",
929               "Last Failure [CSIT-Job-Build-Id]"]
930
931     # Generate the data for the table according to the model in the table
932     # specification
933
934     now = dt.utcnow()
935     timeperiod = timedelta(int(table.get("window", 7)))
936
937     tbl_dict = dict()
938     for job, builds in table["data"].items():
939         for build in builds:
940             build = str(build)
941             for tst_name, tst_data in data[job][build].iteritems():
942                 if tst_name.lower() in table["ignore-list"]:
943                     continue
944                 if tbl_dict.get(tst_name, None) is None:
945                     groups = re.search(REGEX_NIC, tst_data["parent"])
946                     if not groups:
947                         continue
948                     nic = groups.group(0)
949                     tbl_dict[tst_name] = {
950                         "name": "{0}-{1}".format(nic, tst_data["name"]),
951                         "data": OrderedDict()}
952                 try:
953                     generated = input_data.metadata(job, build).\
954                         get("generated", "")
955                     if not generated:
956                         continue
957                     then = dt.strptime(generated, "%Y%m%d %H:%M")
958                     if (now - then) <= timeperiod:
959                         tbl_dict[tst_name]["data"][build] = (
960                             tst_data["status"],
961                             generated,
962                             input_data.metadata(job, build).get("version", ""),
963                             build)
964                 except (TypeError, KeyError) as err:
965                     logging.warning("tst_name: {} - err: {}".
966                                     format(tst_name, repr(err)))
967
968     max_fails = 0
969     tbl_lst = list()
970     for tst_data in tbl_dict.values():
971         fails_nr = 0
972         for val in tst_data["data"].values():
973             if val[0] == "FAIL":
974                 fails_nr += 1
975                 fails_last_date = val[1]
976                 fails_last_vpp = val[2]
977                 fails_last_csit = val[3]
978         if fails_nr:
979             max_fails = fails_nr if fails_nr > max_fails else max_fails
980             tbl_lst.append([tst_data["name"],
981                             fails_nr,
982                             fails_last_date,
983                             fails_last_vpp,
984                             "mrr-daily-build-{0}".format(fails_last_csit)])
985
986     tbl_lst.sort(key=lambda rel: rel[2], reverse=True)
987     tbl_sorted = list()
988     for nrf in range(max_fails, -1, -1):
989         tbl_fails = [item for item in tbl_lst if item[1] == nrf]
990         tbl_sorted.extend(tbl_fails)
991     file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
992
993     logging.info("    Writing file: '{0}'".format(file_name))
994     with open(file_name, "w") as file_handler:
995         file_handler.write(",".join(header) + "\n")
996         for test in tbl_sorted:
997             file_handler.write(",".join([str(item) for item in test]) + '\n')
998
999     txt_file_name = "{0}.txt".format(table["output-file"])
1000     logging.info("    Writing file: '{0}'".format(txt_file_name))
1001     convert_csv_to_pretty_txt(file_name, txt_file_name)
1002
1003
1004 def table_failed_tests_html(table, input_data):
1005     """Generate the table(s) with algorithm: table_failed_tests_html
1006     specified in the specification file.
1007
1008     :param table: Table to generate.
1009     :param input_data: Data to process.
1010     :type table: pandas.Series
1011     :type input_data: InputData
1012     """
1013
1014     testbed = table.get("testbed", None)
1015     if testbed is None:
1016         logging.error("The testbed is not defined for the table '{0}'.".
1017                       format(table.get("title", "")))
1018         return
1019
1020     logging.info("  Generating the table {0} ...".
1021                  format(table.get("title", "")))
1022
1023     try:
1024         with open(table["input-file"], 'rb') as csv_file:
1025             csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
1026             csv_lst = [item for item in csv_content]
1027     except KeyError:
1028         logging.warning("The input file is not defined.")
1029         return
1030     except csv.Error as err:
1031         logging.warning("Not possible to process the file '{0}'.\n{1}".
1032                         format(table["input-file"], err))
1033         return
1034
1035     # Table:
1036     failed_tests = ET.Element("table", attrib=dict(width="100%", border='0'))
1037
1038     # Table header:
1039     tr = ET.SubElement(failed_tests, "tr", attrib=dict(bgcolor="#7eade7"))
1040     for idx, item in enumerate(csv_lst[0]):
1041         alignment = "left" if idx == 0 else "center"
1042         th = ET.SubElement(tr, "th", attrib=dict(align=alignment))
1043         th.text = item
1044
1045     # Rows:
1046     colors = ("#e9f1fb", "#d4e4f7")
1047     for r_idx, row in enumerate(csv_lst[1:]):
1048         background = colors[r_idx % 2]
1049         tr = ET.SubElement(failed_tests, "tr", attrib=dict(bgcolor=background))
1050
1051         # Columns:
1052         for c_idx, item in enumerate(row):
1053             alignment = "left" if c_idx == 0 else "center"
1054             td = ET.SubElement(tr, "td", attrib=dict(align=alignment))
1055             # Name:
1056             if c_idx == 0:
1057                 url = _generate_url("../trending/", testbed, item)
1058                 ref = ET.SubElement(td, "a", attrib=dict(href=url))
1059                 ref.text = item
1060             else:
1061                 td.text = item
1062     try:
1063         with open(table["output-file"], 'w') as html_file:
1064             logging.info("    Writing file: '{0}'".format(table["output-file"]))
1065             html_file.write(".. raw:: html\n\n\t")
1066             html_file.write(ET.tostring(failed_tests))
1067             html_file.write("\n\t<p><br><br></p>\n")
1068     except KeyError:
1069         logging.warning("The output file is not defined.")
1070         return

©2016 FD.io a Linux Foundation Collaborative Project. All Rights Reserved.
Linux Foundation is a registered trademark of The Linux Foundation. Linux is a registered trademark of Linus Torvalds.
Please see our privacy policy and terms of use.