X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=blobdiff_plain;f=resources%2Ftools%2Fpresentation%2Fgenerator_cpta.py;h=2f2ab970e4c58de96cee49e14cad25edd8874d4e;hp=401767042716a4baae8d7fc3d0ccde4b976aa6c6;hb=95de51db79d23e57968e0d27a2a4458326a59639;hpb=b55e324d526e5b05baef015c3614b9743c955992 diff --git a/resources/tools/presentation/generator_cpta.py b/resources/tools/presentation/generator_cpta.py index 4017670427..2f2ab970e4 100644 --- a/resources/tools/presentation/generator_cpta.py +++ b/resources/tools/presentation/generator_cpta.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019 Cisco and/or its affiliates. +# Copyright (c) 2020 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -92,20 +92,31 @@ THEME_OVERRIDES = u"""/* override table width restrictions */ } """ -COLORS = [ - u"SkyBlue", u"Olive", u"Purple", u"Coral", u"Indigo", u"Pink", - u"Chocolate", u"Brown", u"Magenta", u"Cyan", u"Orange", u"Black", - u"Violet", u"Blue", u"Yellow", u"BurlyWood", u"CadetBlue", u"Crimson", - u"DarkBlue", u"DarkCyan", u"DarkGreen", u"Green", u"GoldenRod", - u"LightGreen", u"LightSeaGreen", u"LightSkyBlue", u"Maroon", - u"MediumSeaGreen", u"SeaGreen", u"LightSlateGrey", - u"SkyBlue", u"Olive", u"Purple", u"Coral", u"Indigo", u"Pink", - u"Chocolate", u"Brown", u"Magenta", u"Cyan", u"Orange", u"Black", - u"Violet", u"Blue", u"Yellow", u"BurlyWood", u"CadetBlue", u"Crimson", - u"DarkBlue", u"DarkCyan", u"DarkGreen", u"Green", u"GoldenRod", - u"LightGreen", u"LightSeaGreen", u"LightSkyBlue", u"Maroon", - u"MediumSeaGreen", u"SeaGreen", u"LightSlateGrey" -] +COLORS = ( + u"#1A1110", + u"#DA2647", + u"#214FC6", + u"#01786F", + u"#BD8260", + u"#FFD12A", + u"#A6E7FF", + u"#738276", + u"#C95A49", + u"#FC5A8D", + u"#CEC8EF", + u"#391285", + u"#6F2DA8", + u"#FF878D", + u"#45A27D", + u"#FFD0B9", + u"#FD5240", + u"#DB91EF", + u"#44D7A8", + u"#4F86F7", + u"#84DE02", + u"#FFCFF1", + u"#614051" +) def generate_cpta(spec, data): @@ -146,7 +157,7 @@ def generate_cpta(spec, data): def _generate_trending_traces(in_data, job_name, build_info, - show_trend_line=True, name=u"", color=u""): + name=u"", color=u"", incl_tests=u"MRR"): """Generate the trending traces: - samples, - outliers, regress, progress @@ -155,22 +166,30 @@ def _generate_trending_traces(in_data, job_name, build_info, :param in_data: Full data set. :param job_name: The name of job which generated the data. :param build_info: Information about the builds. - :param show_trend_line: Show moving median (trending plot). :param name: Name of the plot :param color: Name of the color for the plot. + :param incl_tests: Included tests, accepted values: MRR, NDR, PDR :type in_data: OrderedDict :type job_name: str :type build_info: dict - :type show_trend_line: bool :type name: str :type color: str + :type incl_tests: str :returns: Generated traces (list) and the evaluated result. :rtype: tuple(traces, result) """ + if incl_tests not in (u"MRR", u"NDR", u"PDR"): + return list(), None + data_x = list(in_data.keys()) - data_y_pps = list(in_data.values()) - data_y_mpps = [float(item) / 1e6 for item in data_y_pps] + data_y_pps = list() + data_y_mpps = list() + data_y_stdev = list() + for item in in_data.values(): + data_y_pps.append(float(item[u"receive-rate"])) + data_y_stdev.append(float(item[u"receive-stdev"]) / 1e6) + data_y_mpps.append(float(item[u"receive-rate"]) / 1e6) hover_text = list() xaxis = list() @@ -178,28 +197,44 @@ def _generate_trending_traces(in_data, job_name, build_info, str_key = str(key) date = build_info[job_name][str_key][0] hover_str = (u"date: {date}
" - u"value [Mpps]: {value:.3f}
" + u"{property} [Mpps]: {value:.3f}
" + u"" u"{sut}-ref: {build}
" - u"csit-ref: mrr-{period}-build-{build_nr}
" + u"csit-ref: {test}-{period}-build-{build_nr}
" u"testbed: {testbed}") + if incl_tests == u"MRR": + hover_str = hover_str.replace( + u"", f"stdev [Mpps]: {data_y_stdev[index]:.3f}
" + ) + else: + hover_str = hover_str.replace(u"", u"") + if u"-cps" in name: + hover_str = hover_str.replace(u"[Mpps]", u"[Mcps]") if u"dpdk" in job_name: hover_text.append(hover_str.format( date=date, + property=u"average" if incl_tests == u"MRR" else u"throughput", value=data_y_mpps[index], sut=u"dpdk", build=build_info[job_name][str_key][1].rsplit(u'~', 1)[0], + test=incl_tests.lower(), period=u"weekly", build_nr=str_key, testbed=build_info[job_name][str_key][2])) elif u"vpp" in job_name: - hover_text.append(hover_str.format( + hover_str = hover_str.format( date=date, + property=u"average" if incl_tests == u"MRR" else u"throughput", value=data_y_mpps[index], sut=u"vpp", build=build_info[job_name][str_key][1].rsplit(u'~', 1)[0], - period=u"daily", + test=incl_tests.lower(), + period=u"daily" if incl_tests == u"MRR" else u"weekly", build_nr=str_key, - testbed=build_info[job_name][str_key][2])) + testbed=build_info[job_name][str_key][2]) + if u"-cps" in name: + hover_str = hover_str.replace(u"throughput", u"connection rate") + hover_text.append(hover_str) xaxis.append(datetime(int(date[0:4]), int(date[4:6]), int(date[6:8]), int(date[9:11]), int(date[12:]))) @@ -208,8 +243,9 @@ def _generate_trending_traces(in_data, job_name, build_info, for key, value in zip(xaxis, data_y_pps): data_pd[key] = value - anomaly_classification, avgs_pps = classify_anomalies(data_pd) + anomaly_classification, avgs_pps, stdevs_pps = classify_anomalies(data_pd) avgs_mpps = [avg_pps / 1e6 for avg_pps in avgs_pps] + stdevs_mpps = [stdev_pps / 1e6 for stdev_pps in stdevs_pps] anomalies = OrderedDict() anomalies_colors = list() @@ -221,8 +257,7 @@ def _generate_trending_traces(in_data, job_name, build_info, } if anomaly_classification: for index, (key, value) in enumerate(data_pd.items()): - if anomaly_classification[index] in \ - (u"outlier", u"regression", u"progression"): + if anomaly_classification[index] in (u"regression", u"progression"): anomalies[key] = value / 1e6 anomalies_colors.append( anomaly_color[anomaly_classification[index]]) @@ -247,27 +282,34 @@ def _generate_trending_traces(in_data, job_name, build_info, u"symbol": u"circle", }, text=hover_text, - hoverinfo=u"text" + hoverinfo=u"text+name" ) traces = [trace_samples, ] - if show_trend_line: - trace_trend = plgo.Scatter( - x=xaxis, - y=avgs_mpps, - mode=u"lines", - line={ - u"shape": u"linear", - u"width": 1, - u"color": color, - }, - showlegend=False, - legendgroup=name, - name=f"{name}", - text=[f"trend [Mpps]: {avg:.3f}" for avg in avgs_mpps], - hoverinfo=u"text+name" + trend_hover_text = list() + for idx in range(len(data_x)): + trend_hover_str = ( + f"trend [Mpps]: {avgs_mpps[idx]:.3f}
" + f"stdev [Mpps]: {stdevs_mpps[idx]:.3f}" ) - traces.append(trace_trend) + trend_hover_text.append(trend_hover_str) + + trace_trend = plgo.Scatter( + x=xaxis, + y=avgs_mpps, + mode=u"lines", + line={ + u"shape": u"linear", + u"width": 1, + u"color": color, + }, + showlegend=False, + legendgroup=name, + name=f"{name}", + text=trend_hover_text, + hoverinfo=u"text+name" + ) + traces.append(trace_trend) trace_anomalies = plgo.Scatter( x=list(anomalies.keys()), @@ -338,11 +380,9 @@ def _generate_all_charts(spec, input_data): :rtype: dict """ - logs = list() + logging.info(f" Generating the chart {graph.get(u'title', u'')} ...") - logs.append( - (u"INFO", f" Generating the chart {graph.get(u'title', u'')} ...") - ) + incl_tests = graph.get(u"include-tests", u"MRR") job_name = list(graph[u"data"].keys())[0] @@ -350,19 +390,22 @@ def _generate_all_charts(spec, input_data): res = dict() # Transform the data - logs.append( - (u"INFO", - f" Creating the data set for the {graph.get(u'type', u'')} " - f"{graph.get(u'title', u'')}." - ) + logging.info( + f" Creating the data set for the {graph.get(u'type', u'')} " + f"{graph.get(u'title', u'')}." ) if graph.get(u"include", None): data = input_data.filter_tests_by_name( - graph, continue_on_error=True + graph, + params=[u"type", u"result", u"throughput", u"tags"], + continue_on_error=True ) else: - data = input_data.filter_data(graph, continue_on_error=True) + data = input_data.filter_data( + graph, + params=[u"type", u"result", u"throughput", u"tags"], + continue_on_error=True) if data is None or data.empty: logging.error(u"No data.") @@ -378,8 +421,21 @@ def _generate_all_charts(spec, input_data): if chart_data.get(test_name, None) is None: chart_data[test_name] = OrderedDict() try: - chart_data[test_name][int(index)] = \ - test[u"result"][u"receive-rate"] + if incl_tests == u"MRR": + rate = test[u"result"][u"receive-rate"] + stdev = test[u"result"][u"receive-stdev"] + elif incl_tests == u"NDR": + rate = test[u"throughput"][u"NDR"][u"LOWER"] + stdev = float(u"nan") + elif incl_tests == u"PDR": + rate = test[u"throughput"][u"PDR"][u"LOWER"] + stdev = float(u"nan") + else: + continue + chart_data[test_name][int(index)] = { + u"receive-rate": rate, + u"receive-stdev": stdev + } chart_tags[test_name] = test.get(u"tags", None) except (KeyError, TypeError): pass @@ -388,9 +444,12 @@ def _generate_all_charts(spec, input_data): for tst_name, tst_data in chart_data.items(): tst_lst = list() for bld in builds_dict[job_name]: - itm = tst_data.get(int(bld), u'') + itm = tst_data.get(int(bld), dict()) # CSIT-1180: Itm will be list, compute stats. - tst_lst.append(str(itm)) + try: + tst_lst.append(str(itm.get(u"receive-rate", u""))) + except AttributeError: + tst_lst.append(u"") csv_tbl.append(f"{tst_name}," + u",".join(tst_lst) + u'\n') # Generate traces: @@ -405,13 +464,10 @@ def _generate_all_charts(spec, input_data): for tag in group: for tst_name, test_data in chart_data.items(): if not test_data: - logs.append( - (u"WARNING", f"No data for the test {tst_name}") - ) + logging.warning(f"No data for the test {tst_name}") continue if tag not in chart_tags[tst_name]: continue - message = f"index: {index}, test: {tst_name}" try: trace, rslt = _generate_trending_traces( test_data, @@ -419,12 +475,12 @@ def _generate_all_charts(spec, input_data): build_info=build_info, name=u'-'.join(tst_name.split(u'.')[-1]. split(u'-')[2:-1]), - color=COLORS[index]) - except IndexError: - logs.append( - (u"ERROR", f"Out of colors: {message}") + color=COLORS[index], + incl_tests=incl_tests ) - logging.error(f"Out of colors: {message}") + except IndexError: + logging.error(f"Out of colors: index: " + f"{index}, test: {tst_name}") index += 1 continue traces.extend(trace) @@ -436,11 +492,8 @@ def _generate_all_charts(spec, input_data): else: for tst_name, test_data in chart_data.items(): if not test_data: - logs.append( - (u"WARNING", f"No data for the test {tst_name}") - ) + logging.warning(f"No data for the test {tst_name}") continue - message = f"index: {index}, test: {tst_name}" try: trace, rslt = _generate_trending_traces( test_data, @@ -448,10 +501,13 @@ def _generate_all_charts(spec, input_data): build_info=build_info, name=u'-'.join( tst_name.split(u'.')[-1].split(u'-')[2:-1]), - color=COLORS[index]) + color=COLORS[index], + incl_tests=incl_tests + ) except IndexError: - logs.append((u"ERROR", f"Out of colors: {message}")) - logging.error(f"Out of colors: {message}") + logging.error( + f"Out of colors: index: {index}, test: {tst_name}" + ) index += 1 continue traces.extend(trace) @@ -509,25 +565,13 @@ def _generate_all_charts(spec, input_data): f"{spec.cpta[u'output-file']}/{graph[u'output-file-name']}" f"{spec.cpta[u'output-file-type']}") - logs.append((u"INFO", f" Writing the file {name_file} ...")) + logging.info(f" Writing the file {name_file} ...") plpl = plgo.Figure(data=traces, layout=layout) try: ploff.plot(plpl, show_link=False, auto_open=False, filename=name_file) except plerr.PlotlyEmptyDataError: - logs.append((u"WARNING", u"No data for the plot. Skipped.")) - - for level, line in logs: - if level == u"INFO": - logging.info(line) - elif level == u"ERROR": - logging.error(line) - elif level == u"DEBUG": - logging.debug(line) - elif level == u"CRITICAL": - logging.critical(line) - elif level == u"WARNING": - logging.warning(line) + logging.warning(u"No data for the plot. Skipped.") return {u"job_name": job_name, u"csv_table": csv_tbl, u"results": res} @@ -537,7 +581,7 @@ def _generate_all_charts(spec, input_data): builds_dict[job] = list() for build in spec.input[u"builds"][job]: status = build[u"status"] - if status not in (u"failed", u"not found", u"removed"): + if status not in (u"failed", u"not found", u"removed", None): builds_dict[job].append(str(build[u"build"])) # Create "build ID": "date" dict: