X-Git-Url: https://gerrit.fd.io/r/gitweb?p=csit.git;a=blobdiff_plain;f=resources%2Ftools%2Fpresentation%2Fgenerator_tables.py;h=2e7d3dee363cd2126a463d277e14ce99ca7530be;hp=fe0eaaa22e3d62c1cfe4fb2a2199aa372ef55e1b;hb=18f2763797ff782f9068550fc2f01cad4d7056af;hpb=7902281ee08456a4d90bb5829ba42da2c71a6379 diff --git a/resources/tools/presentation/generator_tables.py b/resources/tools/presentation/generator_tables.py index fe0eaaa22e..2e7d3dee36 100644 --- a/resources/tools/presentation/generator_tables.py +++ b/resources/tools/presentation/generator_tables.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Cisco and/or its affiliates. +# Copyright (c) 2022 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -17,6 +17,7 @@ import logging import csv +import math import re from collections import OrderedDict @@ -24,11 +25,11 @@ from xml.etree import ElementTree as ET from datetime import datetime as dt from datetime import timedelta from copy import deepcopy -from json import loads import plotly.graph_objects as go import plotly.offline as ploff import pandas as pd +import prettytable from numpy import nan, isnan from yaml import load, FullLoader, YAMLError @@ -59,7 +60,8 @@ def generate_tables(spec, data): u"table_failed_tests_html": table_failed_tests_html, u"table_oper_data_html": table_oper_data_html, u"table_comparison": table_comparison, - u"table_weekly_comparison": table_weekly_comparison + u"table_weekly_comparison": table_weekly_comparison, + u"table_job_spec_duration": table_job_spec_duration } logging.info(u"Generating the tables ...") @@ -76,6 +78,96 @@ def generate_tables(spec, data): logging.info(u"Done.") +def table_job_spec_duration(table, input_data): + """Generate the table(s) with algorithm: table_job_spec_duration + specified in the specification file. + + :param table: Table to generate. + :param input_data: Data to process. + :type table: pandas.Series + :type input_data: InputData + """ + + _ = input_data + + logging.info(f" Generating the table {table.get(u'title', u'')} ...") + + jb_type = table.get(u"jb-type", None) + + tbl_lst = list() + if jb_type == u"iterative": + for line in table.get(u"lines", tuple()): + tbl_itm = { + u"name": line.get(u"job-spec", u""), + u"data": list() + } + for job, builds in line.get(u"data-set", dict()).items(): + for build_nr in builds: + try: + minutes = input_data.metadata( + job, str(build_nr) + )[u"elapsedtime"] // 60000 + except (KeyError, IndexError, ValueError, AttributeError): + continue + tbl_itm[u"data"].append(minutes) + tbl_itm[u"mean"] = mean(tbl_itm[u"data"]) + tbl_itm[u"stdev"] = stdev(tbl_itm[u"data"]) + tbl_lst.append(tbl_itm) + elif jb_type == u"coverage": + job = table.get(u"data", None) + if not job: + return + for line in table.get(u"lines", tuple()): + try: + tbl_itm = { + u"name": line.get(u"job-spec", u""), + u"mean": input_data.metadata( + list(job.keys())[0], str(line[u"build"]) + )[u"elapsedtime"] // 60000, + u"stdev": float(u"nan") + } + tbl_itm[u"data"] = [tbl_itm[u"mean"], ] + except (KeyError, IndexError, ValueError, AttributeError): + continue + tbl_lst.append(tbl_itm) + else: + logging.warning(f"Wrong type of job-spec: {jb_type}. Skipping.") + return + + for line in tbl_lst: + line[u"mean"] = \ + f"{int(line[u'mean'] // 60):02d}:{int(line[u'mean'] % 60):02d}" + if math.isnan(line[u"stdev"]): + line[u"stdev"] = u"" + else: + line[u"stdev"] = \ + f"{int(line[u'stdev'] //60):02d}:{int(line[u'stdev'] % 60):02d}" + + if not tbl_lst: + return + + rows = list() + for itm in tbl_lst: + rows.append([ + itm[u"name"], + f"{len(itm[u'data'])}", + f"{itm[u'mean']} +- {itm[u'stdev']}" + if itm[u"stdev"] != u"" else f"{itm[u'mean']}" + ]) + + txt_table = prettytable.PrettyTable( + [u"Job Specification", u"Nr of Runs", u"Duration [HH:MM]"] + ) + for row in rows: + txt_table.add_row(row) + txt_table.align = u"r" + txt_table.align[u"Job Specification"] = u"l" + + file_name = f"{table.get(u'output-file', u'')}.txt" + with open(file_name, u"wt", encoding='utf-8') as txt_file: + txt_file.write(str(txt_table)) + + def table_oper_data_html(table, input_data): """Generate the table(s) with algorithm: html_table_oper_data specified in the specification file. @@ -205,28 +297,15 @@ def table_oper_data_html(table, input_data): threads = dict({idx: list() for idx in range(len(runtime))}) for idx, run_data in runtime.items(): for gnode, gdata in run_data.items(): - if gdata[u"vectors"] > 0: - clocks = gdata[u"clocks"] / gdata[u"vectors"] - elif gdata[u"calls"] > 0: - clocks = gdata[u"clocks"] / gdata[u"calls"] - elif gdata[u"suspends"] > 0: - clocks = gdata[u"clocks"] / gdata[u"suspends"] - else: - clocks = 0.0 - if gdata[u"calls"] > 0: - vectors_call = gdata[u"vectors"] / gdata[u"calls"] - else: - vectors_call = 0.0 - if int(gdata[u"calls"]) + int(gdata[u"vectors"]) + \ - int(gdata[u"suspends"]): - threads[idx].append([ - gnode, - int(gdata[u"calls"]), - int(gdata[u"vectors"]), - int(gdata[u"suspends"]), - clocks, - vectors_call - ]) + threads[idx].append([ + gnode, + int(gdata[u"calls"]), + int(gdata[u"vectors"]), + int(gdata[u"suspends"]), + float(gdata[u"clocks"]), + float(gdata[u"vectors"] / gdata[u"calls"]) \ + if gdata[u"calls"] else 0.0 + ]) bold = ET.SubElement(tcol, u"b") bold.text = ( @@ -880,7 +959,7 @@ def table_perf_trending_dash(table, input_data): header = [ u"Test Case", u"Trend [Mpps]", - u"Short-Term Change [%]", + u"Runs [#]", u"Long-Term Change [%]", u"Regressions [#]", u"Progressions [#]" @@ -942,6 +1021,13 @@ def table_perf_trending_dash(table, input_data): last_avg = avgs[-1] avg_week_ago = avgs[max(-win_size, -len(avgs))] + nr_of_last_avgs = 0; + for x in reversed(avgs): + if x == last_avg: + nr_of_last_avgs += 1 + else: + break + if isnan(last_avg) or isnan(avg_week_ago) or avg_week_ago == 0.0: rel_change_last = nan else: @@ -963,28 +1049,23 @@ def table_perf_trending_dash(table, input_data): tbl_lst.append( [tbl_dict[tst_name][u"name"], round(last_avg / 1e6, 2), - rel_change_last, + nr_of_last_avgs, rel_change_long, classification_lst[-win_size+1:].count(u"regression"), classification_lst[-win_size+1:].count(u"progression")]) tbl_lst.sort(key=lambda rel: rel[0]) - tbl_lst.sort(key=lambda rel: rel[3]) tbl_lst.sort(key=lambda rel: rel[2]) - - tbl_sorted = list() - for nrr in range(table[u"window"], -1, -1): - tbl_reg = [item for item in tbl_lst if item[4] == nrr] - for nrp in range(table[u"window"], -1, -1): - tbl_out = [item for item in tbl_reg if item[5] == nrp] - tbl_sorted.extend(tbl_out) + tbl_lst.sort(key=lambda rel: rel[3]) + tbl_lst.sort(key=lambda rel: rel[5], reverse=True) + tbl_lst.sort(key=lambda rel: rel[4], reverse=True) file_name = f"{table[u'output-file']}{table[u'output-file-ext']}" logging.info(f" Writing file: {file_name}") with open(file_name, u"wt") as file_handler: file_handler.write(header_str) - for test in tbl_sorted: + for test in tbl_lst: file_handler.write(u",".join([str(item) for item in test]) + u'\n') logging.info(f" Writing file: {table[u'output-file']}.txt") @@ -1019,6 +1100,8 @@ def _generate_url(testbed, test_name): nic = u"x553" elif u"cx556" in test_name or u"cx556a" in test_name: nic = u"cx556a" + elif u"ena" in test_name: + nic = u"nitro50g" else: nic = u"" @@ -1051,15 +1134,18 @@ def _generate_url(testbed, test_name): cores = u"4t4c" elif u"2t1c" in test_name or \ (u"-1c-" in test_name and - testbed in (u"2n-skx", u"3n-skx", u"2n-clx", u"2n-zn2")): + testbed in + (u"2n-skx", u"3n-skx", u"2n-clx", u"2n-zn2", u"2n-aws", u"3n-aws")): cores = u"2t1c" elif u"4t2c" in test_name or \ (u"-2c-" in test_name and - testbed in (u"2n-skx", u"3n-skx", u"2n-clx", u"2n-zn2")): + testbed in + (u"2n-skx", u"3n-skx", u"2n-clx", u"2n-zn2", u"2n-aws", u"3n-aws")): cores = u"4t2c" elif u"8t4c" in test_name or \ (u"-4c-" in test_name and - testbed in (u"2n-skx", u"3n-skx", u"2n-clx", u"2n-zn2")): + testbed in + (u"2n-skx", u"3n-skx", u"2n-clx", u"2n-zn2", u"2n-aws", u"3n-aws")): cores = u"8t4c" else: cores = u"" @@ -1070,10 +1156,14 @@ def _generate_url(testbed, test_name): driver = u"l3fwd" elif u"avf" in test_name: driver = u"avf" + elif u"af-xdp" in test_name or u"af_xdp" in test_name: + driver = u"af_xdp" elif u"rdma" in test_name: driver = u"rdma" elif u"dnv" in testbed or u"tsh" in testbed: driver = u"ixgbe" + elif u"ena" in test_name: + driver = u"ena" else: driver = u"dpdk" @@ -1178,6 +1268,8 @@ def _generate_url(testbed, test_name): bsf += u"-sw" elif u"hw" in test_name: bsf += u"-hw" + elif u"spe" in test_name: + bsf += u"-spe" elif u"ethip4vxlan" in test_name: domain = u"ip4_tunnels" elif u"ethip4udpgeneve" in test_name: @@ -1374,7 +1466,11 @@ def table_last_failed_tests(table, input_data): if not groups: continue nic = groups.group(0) - failed_tests.append(f"{nic}-{tst_data[u'name']}") + msg = tst_data[u'msg'].replace(u"\n", u"") + msg = re.sub(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})', + 'xxx.xxx.xxx.xxx', msg) + msg = msg.split(u'Also teardown failed')[0] + failed_tests.append(f"{nic}-{tst_data[u'name']}###{msg}") tbl_list.append(passed) tbl_list.append(failed) tbl_list.append(duration) @@ -1809,7 +1905,7 @@ def table_comparison(table, input_data): ) except ZeroDivisionError: break - if delta in (None, float(u"nan"), u"nan", u"NaN"): + if delta is None or math.isnan(delta): break new_row.append({ u"mean": delta * 1e6, @@ -1901,14 +1997,14 @@ def table_comparison(table, input_data): else: if idx < len(cols): new_itm = ( - f"{round(float(itm[u'mean']) / 1e6, 1)} " - f"\u00B1{round(float(itm[u'stdev']) / 1e6, 1)}". + f"{round(float(itm[u'mean']) / 1e6, 2)} " + f"\u00B1{round(float(itm[u'stdev']) / 1e6, 2)}". replace(u"nan", u"NaN") ) else: new_itm = ( - f"{round(float(itm[u'mean']) / 1e6, 1):+} " - f"\u00B1{round(float(itm[u'stdev']) / 1e6, 1)}". + f"{round(float(itm[u'mean']) / 1e6, 2):+} " + f"\u00B1{round(float(itm[u'stdev']) / 1e6, 2)}". replace(u"nan", u"NaN") ) if len(new_itm.rsplit(u" ", 1)[-1]) > max_lens[idx]: