1 # Copyright (c) 2019 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Generation of Continuous Performance Trending and Analysis.
20 import plotly.offline as ploff
21 import plotly.graph_objs as plgo
22 import plotly.exceptions as plerr
24 from collections import OrderedDict
25 from datetime import datetime
26 from copy import deepcopy
28 from utils import archive_input_data, execute_command, classify_anomalies
31 # Command to build the html format of the report
32 HTML_BUILDER = 'sphinx-build -v -c conf_cpta -a ' \
35 '-D version="{date}" ' \
39 # .css file for the html format of the report
40 THEME_OVERRIDES = """/* override table width restrictions */
42 max-width: 1200px !important;
44 .rst-content blockquote {
50 display: inline-block;
58 .wy-menu-vertical li.current a {
60 border-right: solid 1px #c9c9c9;
63 .wy-menu-vertical li.toctree-l2.current > a {
67 .wy-menu-vertical li.toctree-l2.current li.toctree-l3 > a {
72 .wy-menu-vertical li.toctree-l3.current li.toctree-l4 > a {
77 .wy-menu-vertical li.on a, .wy-menu-vertical li.current > a {
84 border-top-width: medium;
85 border-bottom-width: medium;
86 border-top-style: none;
87 border-bottom-style: none;
88 border-top-color: currentcolor;
89 border-bottom-color: currentcolor;
90 padding-left: 2em -4px;
94 COLORS = ["SkyBlue", "Olive", "Purple", "Coral", "Indigo", "Pink",
95 "Chocolate", "Brown", "Magenta", "Cyan", "Orange", "Black",
96 "Violet", "Blue", "Yellow", "BurlyWood", "CadetBlue", "Crimson",
97 "DarkBlue", "DarkCyan", "DarkGreen", "Green", "GoldenRod",
98 "LightGreen", "LightSeaGreen", "LightSkyBlue", "Maroon",
99 "MediumSeaGreen", "SeaGreen", "LightSlateGrey",
100 "SkyBlue", "Olive", "Purple", "Coral", "Indigo", "Pink",
101 "Chocolate", "Brown", "Magenta", "Cyan", "Orange", "Black",
102 "Violet", "Blue", "Yellow", "BurlyWood", "CadetBlue", "Crimson",
103 "DarkBlue", "DarkCyan", "DarkGreen", "Green", "GoldenRod",
104 "LightGreen", "LightSeaGreen", "LightSkyBlue", "Maroon",
105 "MediumSeaGreen", "SeaGreen", "LightSlateGrey"
109 def generate_cpta(spec, data):
110 """Generate all formats and versions of the Continuous Performance Trending
113 :param spec: Specification read from the specification file.
114 :param data: Full data set.
115 :type spec: Specification
116 :type data: InputData
119 logging.info("Generating the Continuous Performance Trending and Analysis "
122 ret_code = _generate_all_charts(spec, data)
124 cmd = HTML_BUILDER.format(
125 date=datetime.utcnow().strftime('%Y-%m-%d %H:%M UTC'),
126 working_dir=spec.environment["paths"]["DIR[WORKING,SRC]"],
127 build_dir=spec.environment["paths"]["DIR[BUILD,HTML]"])
130 with open(spec.environment["paths"]["DIR[CSS_PATCH_FILE]"], "w") as \
132 css_file.write(THEME_OVERRIDES)
134 with open(spec.environment["paths"]["DIR[CSS_PATCH_FILE2]"], "w") as \
136 css_file.write(THEME_OVERRIDES)
138 if spec.configuration.get("archive-inputs", True):
139 archive_input_data(spec)
141 logging.info("Done.")
146 def _generate_trending_traces(in_data, job_name, build_info,
147 show_trend_line=True, name="", color=""):
148 """Generate the trending traces:
150 - outliers, regress, progress
151 - average of normal samples (trending line)
153 :param in_data: Full data set.
154 :param job_name: The name of job which generated the data.
155 :param build_info: Information about the builds.
156 :param show_trend_line: Show moving median (trending plot).
157 :param name: Name of the plot
158 :param color: Name of the color for the plot.
159 :type in_data: OrderedDict
161 :type build_info: dict
162 :type show_trend_line: bool
165 :returns: Generated traces (list) and the evaluated result.
166 :rtype: tuple(traces, result)
169 data_x = list(in_data.keys())
170 data_y = list(in_data.values())
175 date = build_info[job_name][str(idx)][0]
176 hover_str = ("date: {date}<br>"
177 "value: {value:,}<br>"
178 "{sut}-ref: {build}<br>"
179 "csit-ref: mrr-{period}-build-{build_nr}<br>"
180 "testbed: {testbed}")
181 if "dpdk" in job_name:
182 hover_text.append(hover_str.format(
184 value=int(in_data[idx]),
186 build=build_info[job_name][str(idx)][1].rsplit('~', 1)[0],
189 testbed=build_info[job_name][str(idx)][2]))
190 elif "vpp" in job_name:
191 hover_text.append(hover_str.format(
193 value=int(in_data[idx]),
195 build=build_info[job_name][str(idx)][1].rsplit('~', 1)[0],
198 testbed=build_info[job_name][str(idx)][2]))
200 xaxis.append(datetime(int(date[0:4]), int(date[4:6]), int(date[6:8]),
201 int(date[9:11]), int(date[12:])))
203 data_pd = OrderedDict()
204 for key, value in zip(xaxis, data_y):
207 anomaly_classification, avgs = classify_anomalies(data_pd)
209 anomalies = OrderedDict()
210 anomalies_colors = list()
211 anomalies_avgs = list()
217 if anomaly_classification:
218 for idx, (key, value) in enumerate(data_pd.iteritems()):
219 if anomaly_classification[idx] in \
220 ("outlier", "regression", "progression"):
221 anomalies[key] = value
222 anomalies_colors.append(
223 anomaly_color[anomaly_classification[idx]])
224 anomalies_avgs.append(avgs[idx])
225 anomalies_colors.extend([0.0, 0.5, 1.0])
229 trace_samples = plgo.Scatter(
231 y=[y for y in data_y], # Was: y.avg
238 name="{name}".format(name=name),
247 traces = [trace_samples, ]
250 trace_trend = plgo.Scatter(
261 name='{name}'.format(name=name),
262 text=["trend: {0:,}".format(int(avg)) for avg in avgs],
263 hoverinfo="text+name"
265 traces.append(trace_trend)
267 trace_anomalies = plgo.Scatter(
274 name="{name}-anomalies".format(name=name),
277 "symbol": "circle-open",
278 "color": anomalies_colors,
279 "colorscale": [[0.00, "red"],
292 "title": "Circles Marking Data Classification",
293 "titleside": 'right',
298 "tickvals": [0.167, 0.500, 0.833],
299 "ticktext": ["Regression", "Normal", "Progression"],
307 traces.append(trace_anomalies)
309 if anomaly_classification:
310 return traces, anomaly_classification[-1]
315 def _generate_all_charts(spec, input_data):
316 """Generate all charts specified in the specification file.
318 :param spec: Specification.
319 :param input_data: Full data set.
320 :type spec: Specification
321 :type input_data: InputData
324 def _generate_chart(graph):
325 """Generates the chart.
330 logs.append(("INFO", " Generating the chart '{0}' ...".
331 format(graph.get("title", ""))))
333 job_name = graph["data"].keys()[0]
339 logs.append(("INFO", " Creating the data set for the {0} '{1}'.".
340 format(graph.get("type", ""), graph.get("title", ""))))
341 data = input_data.filter_data(graph, continue_on_error=True)
343 logging.error("No data.")
348 for job, job_data in data.iteritems():
351 for index, bld in job_data.items():
352 for test_name, test in bld.items():
353 if chart_data.get(test_name, None) is None:
354 chart_data[test_name] = OrderedDict()
356 chart_data[test_name][int(index)] = \
357 test["result"]["receive-rate"]
358 chart_tags[test_name] = test.get("tags", None)
359 except (KeyError, TypeError):
362 # Add items to the csv table:
363 for tst_name, tst_data in chart_data.items():
365 for bld in builds_dict[job_name]:
366 itm = tst_data.get(int(bld), '')
367 # CSIT-1180: Itm will be list, compute stats.
368 tst_lst.append(str(itm))
369 csv_tbl.append("{0},".format(tst_name) + ",".join(tst_lst) + '\n')
374 groups = graph.get("groups", None)
381 for tst_name, test_data in chart_data.items():
383 logs.append(("WARNING",
384 "No data for the test '{0}'".
387 if tag in chart_tags[tst_name]:
388 message = "index: {index}, test: {test}".format(
389 index=index, test=tst_name)
391 trace, rslt = _generate_trending_traces(
394 build_info=build_info,
395 name='-'.join(tst_name.split('.')[-1].
399 message = "Out of colors: {}".format(message)
400 logs.append(("ERROR", message))
401 logging.error(message)
405 visible.extend([True for _ in range(len(trace))])
409 visibility.append(visible)
411 for tst_name, test_data in chart_data.items():
413 logs.append(("WARNING", "No data for the test '{0}'".
416 message = "index: {index}, test: {test}".format(
417 index=index, test=tst_name)
419 trace, rslt = _generate_trending_traces(
422 build_info=build_info,
423 name='-'.join(tst_name.split('.')[-1].split('-')[2:-1]),
426 message = "Out of colors: {}".format(message)
427 logs.append(("ERROR", message))
428 logging.error(message)
436 # Generate the chart:
438 layout = deepcopy(graph["layout"])
439 except KeyError as err:
440 logging.error("Finished with error: No layout defined")
441 logging.error(repr(err))
445 for i in range(len(visibility)):
447 for r in range(len(visibility)):
448 for _ in range(len(visibility[r])):
449 visible.append(i == r)
456 args=[{"visible": [True for _ in range(len(show[0]))]}, ]
458 for i in range(len(groups)):
460 label = graph["group-names"][i]
461 except (IndexError, KeyError):
462 label = "Group {num}".format(num=i + 1)
466 args=[{"visible": show[i]}, ]
469 layout['updatemenus'] = list([
482 name_file = "{0}-{1}{2}".format(spec.cpta["output-file"],
483 graph["output-file-name"],
484 spec.cpta["output-file-type"])
486 logs.append(("INFO", " Writing the file '{0}' ...".
488 plpl = plgo.Figure(data=traces, layout=layout)
490 ploff.plot(plpl, show_link=False, auto_open=False,
492 except plerr.PlotlyEmptyDataError:
493 logs.append(("WARNING", "No data for the plot. Skipped."))
495 for level, line in logs:
498 elif level == "ERROR":
500 elif level == "DEBUG":
502 elif level == "CRITICAL":
503 logging.critical(line)
504 elif level == "WARNING":
505 logging.warning(line)
507 return {"job_name": job_name, "csv_table": csv_tbl, "results": res}
510 for job in spec.input["builds"].keys():
511 if builds_dict.get(job, None) is None:
512 builds_dict[job] = list()
513 for build in spec.input["builds"][job]:
514 status = build["status"]
515 if status != "failed" and status != "not found" and \
517 builds_dict[job].append(str(build["build"]))
519 # Create "build ID": "date" dict:
521 tb_tbl = spec.environment.get("testbeds", None)
522 for job_name, job_data in builds_dict.items():
523 if build_info.get(job_name, None) is None:
524 build_info[job_name] = OrderedDict()
525 for build in job_data:
527 tb_ip = input_data.metadata(job_name, build).get("testbed", "")
529 testbed = tb_tbl.get(tb_ip, "")
530 build_info[job_name][build] = (
531 input_data.metadata(job_name, build).get("generated", ""),
532 input_data.metadata(job_name, build).get("version", ""),
536 anomaly_classifications = dict()
540 for job_name in builds_dict.keys():
541 if csv_tables.get(job_name, None) is None:
542 csv_tables[job_name] = list()
543 header = "Build Number:," + ",".join(builds_dict[job_name]) + '\n'
544 csv_tables[job_name].append(header)
545 build_dates = [x[0] for x in build_info[job_name].values()]
546 header = "Build Date:," + ",".join(build_dates) + '\n'
547 csv_tables[job_name].append(header)
548 versions = [x[1] for x in build_info[job_name].values()]
549 header = "Version:," + ",".join(versions) + '\n'
550 csv_tables[job_name].append(header)
552 for chart in spec.cpta["plots"]:
553 result = _generate_chart(chart)
555 csv_tables[result["job_name"]].extend(result["csv_table"])
557 if anomaly_classifications.get(result["job_name"], None) is None:
558 anomaly_classifications[result["job_name"]] = dict()
559 anomaly_classifications[result["job_name"]].update(result["results"])
562 for job_name, csv_table in csv_tables.items():
563 file_name = spec.cpta["output-file"] + "-" + job_name + "-trending"
564 with open("{0}.csv".format(file_name), 'w') as file_handler:
565 file_handler.writelines(csv_table)
568 with open("{0}.csv".format(file_name), 'rb') as csv_file:
569 csv_content = csv.reader(csv_file, delimiter=',', quotechar='"')
571 for row in csv_content:
572 if txt_table is None:
573 txt_table = prettytable.PrettyTable(row)
576 for idx, item in enumerate(row):
578 row[idx] = str(round(float(item) / 1000000, 2))
582 txt_table.add_row(row)
583 except Exception as err:
584 logging.warning("Error occurred while generating TXT "
585 "table:\n{0}".format(err))
587 txt_table.align["Build Number:"] = "l"
588 with open("{0}.txt".format(file_name), "w") as txt_file:
589 txt_file.write(str(txt_table))
592 if anomaly_classifications:
594 for job_name, job_data in anomaly_classifications.iteritems():
595 file_name = "{0}-regressions-{1}.txt".\
596 format(spec.cpta["output-file"], job_name)
597 with open(file_name, 'w') as txt_file:
598 for test_name, classification in job_data.iteritems():
599 if classification == "regression":
600 txt_file.write(test_name + '\n')
601 if classification == "regression" or \
602 classification == "outlier":
604 file_name = "{0}-progressions-{1}.txt".\
605 format(spec.cpta["output-file"], job_name)
606 with open(file_name, 'w') as txt_file:
607 for test_name, classification in job_data.iteritems():
608 if classification == "progression":
609 txt_file.write(test_name + '\n')
613 logging.info("Partial results: {0}".format(anomaly_classifications))
614 logging.info("Result: {0}".format(result))