1 # Copyright (c) 2020 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
14 """Generation of Continuous Performance Trending and Analysis.
20 from collections import OrderedDict
21 from datetime import datetime
22 from copy import deepcopy
25 import plotly.offline as ploff
26 import plotly.graph_objs as plgo
27 import plotly.exceptions as plerr
29 from pal_utils import archive_input_data, execute_command, classify_anomalies
32 # Command to build the html format of the report
33 HTML_BUILDER = u'sphinx-build -v -c conf_cpta -a ' \
36 u'-D version="{date}" ' \
40 # .css file for the html format of the report
41 THEME_OVERRIDES = u"""/* override table width restrictions */
43 max-width: 1200px !important;
45 .rst-content blockquote {
51 display: inline-block;
59 .wy-menu-vertical li.current a {
61 border-right: solid 1px #c9c9c9;
64 .wy-menu-vertical li.toctree-l2.current > a {
68 .wy-menu-vertical li.toctree-l2.current li.toctree-l3 > a {
73 .wy-menu-vertical li.toctree-l3.current li.toctree-l4 > a {
78 .wy-menu-vertical li.on a, .wy-menu-vertical li.current > a {
85 border-top-width: medium;
86 border-bottom-width: medium;
87 border-top-style: none;
88 border-bottom-style: none;
89 border-top-color: currentcolor;
90 border-bottom-color: currentcolor;
91 padding-left: 2em -4px;
96 u"SkyBlue", u"Olive", u"Purple", u"Coral", u"Indigo", u"Pink",
97 u"Chocolate", u"Brown", u"Magenta", u"Cyan", u"Orange", u"Black",
98 u"Violet", u"Blue", u"Yellow", u"BurlyWood", u"CadetBlue", u"Crimson",
99 u"DarkBlue", u"DarkCyan", u"DarkGreen", u"Green", u"GoldenRod",
100 u"LightGreen", u"LightSeaGreen", u"LightSkyBlue", u"Maroon",
101 u"MediumSeaGreen", u"SeaGreen", u"LightSlateGrey",
102 u"SkyBlue", u"Olive", u"Purple", u"Coral", u"Indigo", u"Pink",
103 u"Chocolate", u"Brown", u"Magenta", u"Cyan", u"Orange", u"Black",
104 u"Violet", u"Blue", u"Yellow", u"BurlyWood", u"CadetBlue", u"Crimson",
105 u"DarkBlue", u"DarkCyan", u"DarkGreen", u"Green", u"GoldenRod",
106 u"LightGreen", u"LightSeaGreen", u"LightSkyBlue", u"Maroon",
107 u"MediumSeaGreen", u"SeaGreen", u"LightSlateGrey"
111 def generate_cpta(spec, data):
112 """Generate all formats and versions of the Continuous Performance Trending
115 :param spec: Specification read from the specification file.
116 :param data: Full data set.
117 :type spec: Specification
118 :type data: InputData
121 logging.info(u"Generating the Continuous Performance Trending and Analysis "
124 ret_code = _generate_all_charts(spec, data)
126 cmd = HTML_BUILDER.format(
127 date=datetime.utcnow().strftime(u'%Y-%m-%d %H:%M UTC'),
128 working_dir=spec.environment[u'paths'][u'DIR[WORKING,SRC]'],
129 build_dir=spec.environment[u'paths'][u'DIR[BUILD,HTML]'])
132 with open(spec.environment[u'paths'][u'DIR[CSS_PATCH_FILE]'], u'w') as \
134 css_file.write(THEME_OVERRIDES)
136 with open(spec.environment[u'paths'][u'DIR[CSS_PATCH_FILE2]'], u'w') as \
138 css_file.write(THEME_OVERRIDES)
140 if spec.configuration.get(u"archive-inputs", True):
141 archive_input_data(spec)
143 logging.info(u"Done.")
148 def _generate_trending_traces(in_data, job_name, build_info,
149 show_trend_line=True, name=u"", color=u""):
150 """Generate the trending traces:
152 - outliers, regress, progress
153 - average of normal samples (trending line)
155 :param in_data: Full data set.
156 :param job_name: The name of job which generated the data.
157 :param build_info: Information about the builds.
158 :param show_trend_line: Show moving median (trending plot).
159 :param name: Name of the plot
160 :param color: Name of the color for the plot.
161 :type in_data: OrderedDict
163 :type build_info: dict
164 :type show_trend_line: bool
167 :returns: Generated traces (list) and the evaluated result.
168 :rtype: tuple(traces, result)
171 data_x = list(in_data.keys())
172 data_y_pps = list(in_data.values())
173 data_y_mpps = [float(item) / 1e6 for item in data_y_pps]
177 for index, key in enumerate(data_x):
179 date = build_info[job_name][str_key][0]
180 hover_str = (u"date: {date}<br>"
181 u"value [Mpps]: {value:.3f}<br>"
182 u"{sut}-ref: {build}<br>"
183 u"csit-ref: mrr-{period}-build-{build_nr}<br>"
184 u"testbed: {testbed}")
185 if u"dpdk" in job_name:
186 hover_text.append(hover_str.format(
188 value=data_y_mpps[index],
190 build=build_info[job_name][str_key][1].rsplit(u'~', 1)[0],
193 testbed=build_info[job_name][str_key][2]))
194 elif u"vpp" in job_name:
195 hover_text.append(hover_str.format(
197 value=data_y_mpps[index],
199 build=build_info[job_name][str_key][1].rsplit(u'~', 1)[0],
202 testbed=build_info[job_name][str_key][2]))
204 xaxis.append(datetime(int(date[0:4]), int(date[4:6]), int(date[6:8]),
205 int(date[9:11]), int(date[12:])))
207 data_pd = OrderedDict()
208 for key, value in zip(xaxis, data_y_pps):
211 anomaly_classification, avgs_pps = classify_anomalies(data_pd)
212 avgs_mpps = [avg_pps / 1e6 for avg_pps in avgs_pps]
214 anomalies = OrderedDict()
215 anomalies_colors = list()
216 anomalies_avgs = list()
222 if anomaly_classification:
223 for index, (key, value) in enumerate(data_pd.items()):
224 if anomaly_classification[index] in \
225 (u"outlier", u"regression", u"progression"):
226 anomalies[key] = value / 1e6
227 anomalies_colors.append(
228 anomaly_color[anomaly_classification[index]])
229 anomalies_avgs.append(avgs_mpps[index])
230 anomalies_colors.extend([0.0, 0.5, 1.0])
234 trace_samples = plgo.Scatter(
247 u"symbol": u"circle",
252 traces = [trace_samples, ]
255 trace_trend = plgo.Scatter(
267 text=[f"trend [Mpps]: {avg:.3f}" for avg in avgs_mpps],
268 hoverinfo=u"text+name"
270 traces.append(trace_trend)
272 trace_anomalies = plgo.Scatter(
273 x=list(anomalies.keys()),
279 name=f"{name}-anomalies",
282 u"symbol": u"circle-open",
283 u"color": anomalies_colors,
299 u"title": u"Circles Marking Data Classification",
300 u"titleside": u"right",
304 u"tickmode": u"array",
305 u"tickvals": [0.167, 0.500, 0.833],
306 u"ticktext": [u"Regression", u"Normal", u"Progression"],
314 traces.append(trace_anomalies)
316 if anomaly_classification:
317 return traces, anomaly_classification[-1]
322 def _generate_all_charts(spec, input_data):
323 """Generate all charts specified in the specification file.
325 :param spec: Specification.
326 :param input_data: Full data set.
327 :type spec: Specification
328 :type input_data: InputData
331 def _generate_chart(graph):
332 """Generates the chart.
334 :param graph: The graph to be generated
336 :returns: Dictionary with the job name, csv table with results and
337 list of tests classification results.
344 (u"INFO", f" Generating the chart {graph.get(u'title', u'')} ...")
347 job_name = list(graph[u"data"].keys())[0]
355 f" Creating the data set for the {graph.get(u'type', u'')} "
356 f"{graph.get(u'title', u'')}."
360 if graph.get(u"include", None):
361 data = input_data.filter_tests_by_name(
363 params=[u"type", u"result", u"tags"],
364 continue_on_error=True
367 data = input_data.filter_data(
369 params=[u"type", u"result", u"tags"],
370 continue_on_error=True)
372 if data is None or data.empty:
373 logging.error(u"No data.")
378 for job, job_data in data.items():
381 for index, bld in job_data.items():
382 for test_name, test in bld.items():
383 if chart_data.get(test_name, None) is None:
384 chart_data[test_name] = OrderedDict()
386 chart_data[test_name][int(index)] = \
387 test[u"result"][u"receive-rate"]
388 chart_tags[test_name] = test.get(u"tags", None)
389 except (KeyError, TypeError):
392 # Add items to the csv table:
393 for tst_name, tst_data in chart_data.items():
395 for bld in builds_dict[job_name]:
396 itm = tst_data.get(int(bld), u'')
397 # CSIT-1180: Itm will be list, compute stats.
398 tst_lst.append(str(itm))
399 csv_tbl.append(f"{tst_name}," + u",".join(tst_lst) + u'\n')
404 groups = graph.get(u"groups", None)
411 for tst_name, test_data in chart_data.items():
414 (u"WARNING", f"No data for the test {tst_name}")
417 if tag not in chart_tags[tst_name]:
419 message = f"index: {index}, test: {tst_name}"
421 trace, rslt = _generate_trending_traces(
424 build_info=build_info,
425 name=u'-'.join(tst_name.split(u'.')[-1].
430 (u"ERROR", f"Out of colors: {message}")
432 logging.error(f"Out of colors: {message}")
436 visible.extend([True for _ in range(len(trace))])
440 visibility.append(visible)
442 for tst_name, test_data in chart_data.items():
445 (u"WARNING", f"No data for the test {tst_name}")
448 message = f"index: {index}, test: {tst_name}"
450 trace, rslt = _generate_trending_traces(
453 build_info=build_info,
455 tst_name.split(u'.')[-1].split(u'-')[2:-1]),
458 logs.append((u"ERROR", f"Out of colors: {message}"))
459 logging.error(f"Out of colors: {message}")
467 # Generate the chart:
469 layout = deepcopy(graph[u"layout"])
470 except KeyError as err:
471 logging.error(u"Finished with error: No layout defined")
472 logging.error(repr(err))
476 for i in range(len(visibility)):
478 for vis_idx, _ in enumerate(visibility):
479 for _ in range(len(visibility[vis_idx])):
480 visible.append(i == vis_idx)
487 args=[{u"visible": [True for _ in range(len(show[0]))]}, ]
489 for i in range(len(groups)):
491 label = graph[u"group-names"][i]
492 except (IndexError, KeyError):
493 label = f"Group {i + 1}"
497 args=[{u"visible": show[i]}, ]
500 layout[u"updatemenus"] = list([
514 f"{spec.cpta[u'output-file']}/{graph[u'output-file-name']}"
515 f"{spec.cpta[u'output-file-type']}")
517 logs.append((u"INFO", f" Writing the file {name_file} ..."))
518 plpl = plgo.Figure(data=traces, layout=layout)
520 ploff.plot(plpl, show_link=False, auto_open=False,
522 except plerr.PlotlyEmptyDataError:
523 logs.append((u"WARNING", u"No data for the plot. Skipped."))
525 for level, line in logs:
528 elif level == u"ERROR":
530 elif level == u"DEBUG":
532 elif level == u"CRITICAL":
533 logging.critical(line)
534 elif level == u"WARNING":
535 logging.warning(line)
537 return {u"job_name": job_name, u"csv_table": csv_tbl, u"results": res}
540 for job in spec.input[u"builds"].keys():
541 if builds_dict.get(job, None) is None:
542 builds_dict[job] = list()
543 for build in spec.input[u"builds"][job]:
544 status = build[u"status"]
545 if status not in (u"failed", u"not found", u"removed"):
546 builds_dict[job].append(str(build[u"build"]))
548 # Create "build ID": "date" dict:
550 tb_tbl = spec.environment.get(u"testbeds", None)
551 for job_name, job_data in builds_dict.items():
552 if build_info.get(job_name, None) is None:
553 build_info[job_name] = OrderedDict()
554 for build in job_data:
556 tb_ip = input_data.metadata(job_name, build).get(u"testbed", u"")
558 testbed = tb_tbl.get(tb_ip, u"")
559 build_info[job_name][build] = (
560 input_data.metadata(job_name, build).get(u"generated", u""),
561 input_data.metadata(job_name, build).get(u"version", u""),
565 anomaly_classifications = dict()
567 # Create the table header:
569 for job_name in builds_dict:
570 if csv_tables.get(job_name, None) is None:
571 csv_tables[job_name] = list()
572 header = f"Build Number:,{u','.join(builds_dict[job_name])}\n"
573 csv_tables[job_name].append(header)
574 build_dates = [x[0] for x in build_info[job_name].values()]
575 header = f"Build Date:,{u','.join(build_dates)}\n"
576 csv_tables[job_name].append(header)
577 versions = [x[1] for x in build_info[job_name].values()]
578 header = f"Version:,{u','.join(versions)}\n"
579 csv_tables[job_name].append(header)
581 for chart in spec.cpta[u"plots"]:
582 result = _generate_chart(chart)
586 csv_tables[result[u"job_name"]].extend(result[u"csv_table"])
588 if anomaly_classifications.get(result[u"job_name"], None) is None:
589 anomaly_classifications[result[u"job_name"]] = dict()
590 anomaly_classifications[result[u"job_name"]].update(result[u"results"])
593 for job_name, csv_table in csv_tables.items():
594 file_name = f"{spec.cpta[u'output-file']}/{job_name}-trending"
595 with open(f"{file_name}.csv", u"wt") as file_handler:
596 file_handler.writelines(csv_table)
599 with open(f"{file_name}.csv", u"rt") as csv_file:
600 csv_content = csv.reader(csv_file, delimiter=u',', quotechar=u'"')
602 for row in csv_content:
603 if txt_table is None:
604 txt_table = prettytable.PrettyTable(row)
607 for idx, item in enumerate(row):
609 row[idx] = str(round(float(item) / 1000000, 2))
613 txt_table.add_row(row)
614 # PrettyTable raises Exception
615 except Exception as err:
617 f"Error occurred while generating TXT table:\n{err}"
620 txt_table.align[u"Build Number:"] = u"l"
621 with open(f"{file_name}.txt", u"wt") as txt_file:
622 txt_file.write(str(txt_table))
625 if anomaly_classifications:
627 for job_name, job_data in anomaly_classifications.items():
629 f"{spec.cpta[u'output-file']}/regressions-{job_name}.txt"
630 with open(file_name, u'w') as txt_file:
631 for test_name, classification in job_data.items():
632 if classification == u"regression":
633 txt_file.write(test_name + u'\n')
634 if classification in (u"regression", u"outlier"):
637 f"{spec.cpta[u'output-file']}/progressions-{job_name}.txt"
638 with open(file_name, u'w') as txt_file:
639 for test_name, classification in job_data.items():
640 if classification == u"progression":
641 txt_file.write(test_name + u'\n')
645 logging.info(f"Partial results: {anomaly_classifications}")
646 logging.info(f"Result: {result}")